[llvm] [WIP][X86] Always use 128-bit V_SET0/AVX512_128_SET0 patterns, along with SUBREG_TO_REG for extension to 256/512-bit vectors (PR #212950)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 02:40:31 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/212950
>From a5f71bdd31d89b1bf691ce02ba77192367534dca Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Wed, 29 Jul 2026 09:30:46 +0100
Subject: [PATCH] [X86] Always use 128-bit V_SET0/AVX512_128_SET0 patterns,
along with SUBREG_TO_REG for extension to 256/512-bit vectors
---
llvm/lib/Target/X86/X86FastPreTileConfig.cpp | 12 +-
llvm/lib/Target/X86/X86InstrAVX512.td | 43 +-
llvm/lib/Target/X86/X86InstrInfo.cpp | 51 +-
llvm/lib/Target/X86/X86InstrSSE.td | 27 +-
llvm/lib/Target/X86/X86PreTileConfig.cpp | 12 +-
.../MIR/X86/machine-verifier-address.mir | 7 +-
.../test/CodeGen/X86/2012-01-12-extract-sv.ll | 3 +-
llvm/test/CodeGen/X86/AMX/amx-across-func.ll | 3 +
llvm/test/CodeGen/X86/AMX/amx-configO0toO0.ll | 1 +
llvm/test/CodeGen/X86/AMX/amx-configO2toO0.ll | 1 +
.../CodeGen/X86/AMX/amx-fastconfig-spill.mir | 10 +-
llvm/test/CodeGen/X86/AMX/amx-fastconfig.mir | 5 +-
.../CodeGen/X86/AMX/amx-fastpreconfig.mir | 5 +-
.../X86/AMX/amx-greedy-ra-spill-shape.ll | 4 +-
llvm/test/CodeGen/X86/AMX/amx-greedy-ra.ll | 30 +-
.../X86/AMX/amx-sink-config-after-calls.mir | 3 +-
llvm/test/CodeGen/X86/AMX/amx-zero-config.ll | 2 +
.../CodeGen/X86/amx-across-func-tilemovrow.ll | 1 +
llvm/test/CodeGen/X86/apx/kmov-isel.ll | 2 +-
llvm/test/CodeGen/X86/avx-select.ll | 36 +-
.../CodeGen/X86/avx2-intrinsics-fast-isel.ll | 14 +-
llvm/test/CodeGen/X86/avx512-build-vector.ll | 6 +-
...avx512-gather-scatter-intrin-deprecated.ll | 4 +-
.../X86/avx512-gather-scatter-intrin.ll | 8 +-
.../test/CodeGen/X86/avx512-insert-extract.ll | 28 +-
llvm/test/CodeGen/X86/avx512-mask-op.ll | 104 ++--
llvm/test/CodeGen/X86/avx512-mask-set-opt.ll | 2 +-
llvm/test/CodeGen/X86/avx512-select.ll | 36 +-
llvm/test/CodeGen/X86/avx512-vec-cmp.ll | 2 +-
llvm/test/CodeGen/X86/avx512-vselect.ll | 28 +-
llvm/test/CodeGen/X86/avx512f-256-set0.mir | 8 +-
llvm/test/CodeGen/X86/bit-manip-i256.ll | 144 ++---
llvm/test/CodeGen/X86/bit-manip-i512.ll | 538 +++++++++---------
.../X86/bitcast-int-to-vector-bool-sext.ll | 1 -
llvm/test/CodeGen/X86/combine-icmp.ll | 5 +-
llvm/test/CodeGen/X86/combine-pmuldq.ll | 16 +-
.../test/CodeGen/X86/combine-storetomstore.ll | 3 +
llvm/test/CodeGen/X86/dpbusd.ll | 18 +-
llvm/test/CodeGen/X86/dpbusd_const.ll | 24 +-
llvm/test/CodeGen/X86/gfni-funnel-shifts.ll | 4 +-
llvm/test/CodeGen/X86/gfni-rotates.ll | 8 +-
.../CodeGen/X86/insert-subvector-broadcast.ll | 10 +-
llvm/test/CodeGen/X86/kshift.ll | 54 +-
llvm/test/CodeGen/X86/madd.ll | 60 +-
llvm/test/CodeGen/X86/masked_gather.ll | 2 +-
.../test/CodeGen/X86/masked_gather_scatter.ll | 94 +--
.../X86/masked_gather_scatter_widen.ll | 22 +-
.../CodeGen/X86/masked_loadstore_split.ll | 7 +-
.../CodeGen/X86/min-legal-vector-width.ll | 14 +-
.../X86/non-foldable-with-the-same-mask.mir | 104 ++--
llvm/test/CodeGen/X86/pr174871.ll | 5 +-
llvm/test/CodeGen/X86/pr34592.ll | 1 +
llvm/test/CodeGen/X86/pr45067.ll | 6 +-
llvm/test/CodeGen/X86/pr51615.ll | 16 +-
llvm/test/CodeGen/X86/pr53842.ll | 21 +-
llvm/test/CodeGen/X86/pr63108.ll | 4 +-
llvm/test/CodeGen/X86/prefer-avx256-lzcnt.ll | 2 +-
llvm/test/CodeGen/X86/sadd_sat_vec.ll | 56 +-
llvm/test/CodeGen/X86/shift-i256.ll | 68 +--
llvm/test/CodeGen/X86/shift-i512.ll | 184 +++---
llvm/test/CodeGen/X86/shuffle-half.ll | 9 +-
llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll | 10 +-
.../X86/speculative-load-hardening-gather.ll | 4 +-
llvm/test/CodeGen/X86/vec_int_to_fp.ll | 32 +-
llvm/test/CodeGen/X86/vec_saddo.ll | 88 +--
llvm/test/CodeGen/X86/vector-extend-inreg.ll | 4 +-
llvm/test/CodeGen/X86/vector-fshl-256.ll | 2 +-
llvm/test/CodeGen/X86/vector-fshl-512.ll | 8 +-
llvm/test/CodeGen/X86/vector-fshl-rot-256.ll | 14 +-
llvm/test/CodeGen/X86/vector-fshl-rot-512.ll | 8 +-
llvm/test/CodeGen/X86/vector-fshr-256.ll | 2 +-
llvm/test/CodeGen/X86/vector-fshr-512.ll | 48 +-
llvm/test/CodeGen/X86/vector-fshr-rot-256.ll | 14 +-
llvm/test/CodeGen/X86/vector-fshr-rot-512.ll | 8 +-
llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll | 42 +-
.../vector-interleaved-load-i16-stride-4.ll | 490 ++++++++--------
llvm/test/CodeGen/X86/vector-lzcnt-256.ll | 404 ++++++-------
.../CodeGen/X86/vector-popcnt-256-ult-ugt.ll | 360 ++++++------
llvm/test/CodeGen/X86/vector-popcnt-256.ll | 4 +-
.../CodeGen/X86/vector-popcnt-512-ult-ugt.ll | 360 ++++++------
llvm/test/CodeGen/X86/vector-popcnt-512.ll | 4 +-
.../CodeGen/X86/vector-reduce-add-mask.ll | 4 +-
.../CodeGen/X86/vector-reduce-add-sext.ll | 39 +-
.../CodeGen/X86/vector-reduce-add-zext.ll | 20 +-
llvm/test/CodeGen/X86/vector-reduce-ctpop.ll | 159 ++++--
.../test/CodeGen/X86/vector-reduce-or-bool.ll | 20 +-
.../CodeGen/X86/vector-reduce-xor-bool.ll | 24 +-
llvm/test/CodeGen/X86/vector-rotate-256.ll | 14 +-
llvm/test/CodeGen/X86/vector-rotate-512.ll | 8 +-
.../test/CodeGen/X86/vector-shift-lshr-512.ll | 20 +-
.../CodeGen/X86/vector-shuffle-256-v16.ll | 6 +-
.../CodeGen/X86/vector-shuffle-256-v32.ll | 12 +-
.../test/CodeGen/X86/vector-shuffle-avx512.ll | 6 +-
llvm/test/CodeGen/X86/vector-shuffle-v1.ll | 6 +-
llvm/test/CodeGen/X86/vmaskmov-offset.ll | 14 +-
...lar-shift-by-byte-multiple-legalization.ll | 235 ++++----
.../CodeGen/X86/zero-call-used-regs-simd.ll | 5 -
.../CodeGen/X86/zero_extend_vector_inreg.ll | 80 ++-
.../zero_extend_vector_inreg_of_broadcast.ll | 86 ++-
...d_vector_inreg_of_broadcast_from_memory.ll | 437 +++++++-------
100 files changed, 2549 insertions(+), 2550 deletions(-)
diff --git a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
index 9efe335666ac2..b18c16f757a40 100644
--- a/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86FastPreTileConfig.cpp
@@ -175,13 +175,21 @@ void X86FastPreTileConfigImpl::InitializeTileConfigStackSpace() {
MachineInstr *MI = &*MBB.getFirstNonPHI();
DebugLoc DL;
if (ST->hasAVX512()) {
+ Register Xmm = MRI->createVirtualRegister(&X86::VR128XRegClass);
+ BuildMI(MBB, MI, DL, TII->get(X86::AVX512_128_SET0), Xmm);
Register Zmm = MRI->createVirtualRegister(&X86::VR512RegClass);
- BuildMI(MBB, MI, DL, TII->get(X86::AVX512_512_SET0), Zmm);
+ BuildMI(MBB, MI, DL, TII->get(X86::SUBREG_TO_REG), Zmm)
+ .addReg(Xmm)
+ .addImm(X86::sub_xmm);
addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr)), CfgSS)
.addReg(Zmm);
} else if (ST->hasAVX2()) {
+ Register Xmm = MRI->createVirtualRegister(&X86::VR128RegClass);
+ BuildMI(MBB, MI, DL, TII->get(X86::V_SET0), Xmm);
Register Ymm = MRI->createVirtualRegister(&X86::VR256RegClass);
- BuildMI(MBB, MI, DL, TII->get(X86::AVX_SET0), Ymm);
+ BuildMI(MBB, MI, DL, TII->get(X86::SUBREG_TO_REG), Ymm)
+ .addReg(Xmm)
+ .addImm(X86::sub_xmm);
addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSYmr)), CfgSS)
.addReg(Ymm);
addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSYmr)), CfgSS,
diff --git a/llvm/lib/Target/X86/X86InstrAVX512.td b/llvm/lib/Target/X86/X86InstrAVX512.td
index d2916346e7d33..9fc27c5aff13c 100644
--- a/llvm/lib/Target/X86/X86InstrAVX512.td
+++ b/llvm/lib/Target/X86/X86InstrAVX512.td
@@ -296,8 +296,8 @@ multiclass AVX512_maskable_fma<bits<8> O, Format F, X86VectorVTInfo _,
// load of an all-zeros value if folding it would be beneficial.
let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
isPseudo = 1, Predicates = [HasAVX512], SchedRW = [WriteZero] in {
-def AVX512_512_SET0 : I<0, Pseudo, (outs VR512:$dst), (ins), "",
- [(set VR512:$dst, (v16i32 immAllZerosV))]>;
+def AVX512_128_SET0 : I<0, Pseudo, (outs VR128X:$dst), (ins), "",
+ [(set VR128X:$dst, (v4i32 immAllZerosV))]>;
def AVX512_512_SETALLONES : I<0, Pseudo, (outs VR512:$dst), (ins), "",
[(set VR512:$dst, (v16i32 immAllOnesV))]>;
let AddedComplexity = 1, Predicates = [HasVLX] in {
@@ -308,15 +308,6 @@ let AddedComplexity = 1, Predicates = [HasVLX] in {
}
}
-let Predicates = [HasAVX512] in {
-def : Pat<(v64i8 immAllZerosV), (AVX512_512_SET0)>;
-def : Pat<(v32i16 immAllZerosV), (AVX512_512_SET0)>;
-def : Pat<(v8i64 immAllZerosV), (AVX512_512_SET0)>;
-def : Pat<(v32f16 immAllZerosV), (AVX512_512_SET0)>;
-def : Pat<(v16f32 immAllZerosV), (AVX512_512_SET0)>;
-def : Pat<(v8f64 immAllZerosV), (AVX512_512_SET0)>;
-}
-
// Alias instructions that allow VPTERNLOG to be used with a mask to create
// a mix of all ones and all zeros elements. This is done this way to force
// the same register to be used as input for all three sources.
@@ -333,14 +324,6 @@ def AVX512_512_SEXT_MASK_64 : I<0, Pseudo, (outs VR512:$dst),
(v8i64 immAllZerosV)))]>;
}
-let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
- isPseudo = 1, Predicates = [HasAVX512], SchedRW = [WriteZero] in {
-def AVX512_128_SET0 : I<0, Pseudo, (outs VR128X:$dst), (ins), "",
- [(set VR128X:$dst, (v4i32 immAllZerosV))]>;
-def AVX512_256_SET0 : I<0, Pseudo, (outs VR256X:$dst), (ins), "",
- [(set VR256X:$dst, (v8i32 immAllZerosV))]>;
-}
-
let Predicates = [HasAVX512] in {
def : Pat<(v8i16 immAllZerosV), (AVX512_128_SET0)>;
def : Pat<(v16i8 immAllZerosV), (AVX512_128_SET0)>;
@@ -348,12 +331,22 @@ def : Pat<(v2i64 immAllZerosV), (AVX512_128_SET0)>;
def : Pat<(v8f16 immAllZerosV), (AVX512_128_SET0)>;
def : Pat<(v4f32 immAllZerosV), (AVX512_128_SET0)>;
def : Pat<(v2f64 immAllZerosV), (AVX512_128_SET0)>;
-def : Pat<(v32i8 immAllZerosV), (AVX512_256_SET0)>;
-def : Pat<(v16i16 immAllZerosV), (AVX512_256_SET0)>;
-def : Pat<(v4i64 immAllZerosV), (AVX512_256_SET0)>;
-def : Pat<(v16f16 immAllZerosV), (AVX512_256_SET0)>;
-def : Pat<(v8f32 immAllZerosV), (AVX512_256_SET0)>;
-def : Pat<(v4f64 immAllZerosV), (AVX512_256_SET0)>;
+
+def : Pat<(v32i8 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+def : Pat<(v16i16 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+def : Pat<(v8i32 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+def : Pat<(v4i64 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+def : Pat<(v16f16 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+def : Pat<(v8f32 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+def : Pat<(v4f64 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+
+def : Pat<(v64i8 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+def : Pat<(v32i16 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+def : Pat<(v16i32 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+def : Pat<(v8i64 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+def : Pat<(v32f16 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+def : Pat<(v16f32 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
+def : Pat<(v8f64 immAllZerosV), (SUBREG_TO_REG (v4i32 (AVX512_128_SET0)), sub_xmm)>;
}
// Alias instructions that map fld0 to xorps for sse or vxorps for avx.
diff --git a/llvm/lib/Target/X86/X86InstrInfo.cpp b/llvm/lib/Target/X86/X86InstrInfo.cpp
index f41a68d7d17e0..e2c3a156290ef 100644
--- a/llvm/lib/Target/X86/X86InstrInfo.cpp
+++ b/llvm/lib/Target/X86/X86InstrInfo.cpp
@@ -782,8 +782,6 @@ bool X86InstrInfo::isReMaterializableImpl(
case X86::AVX1_SETALLONES:
case X86::AVX2_SETALLONES:
case X86::AVX512_128_SET0:
- case X86::AVX512_256_SET0:
- case X86::AVX512_512_SET0:
case X86::AVX512_128_SETALLONES:
case X86::AVX512_256_SETALLONES:
case X86::AVX512_512_SETALLONES:
@@ -791,7 +789,6 @@ bool X86InstrInfo::isReMaterializableImpl(
case X86::AVX512_FsFLD0SH:
case X86::AVX512_FsFLD0SS:
case X86::AVX512_FsFLD0F128:
- case X86::AVX_SET0:
case X86::FsFLD0SD:
case X86::FsFLD0SS:
case X86::FsFLD0SH:
@@ -6337,16 +6334,6 @@ bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
case X86::FsFLD0SH:
case X86::FsFLD0F128:
return Expand2AddrUndef(MIB, get(HasAVX ? X86::VXORPSrr : X86::XORPSrr));
- case X86::AVX_SET0: {
- assert(HasAVX && "AVX not supported");
- const TargetRegisterInfo *TRI = &getRegisterInfo();
- Register SrcReg = MIB.getReg(0);
- Register XReg = TRI->getSubReg(SrcReg, X86::sub_xmm);
- MIB->getOperand(0).setReg(XReg);
- Expand2AddrUndef(MIB, get(X86::VXORPSrr));
- MIB.addReg(SrcReg, RegState::ImplicitDefine);
- return true;
- }
case X86::AVX512_128_SET0:
case X86::AVX512_FsFLD0SH:
case X86::AVX512_FsFLD0SS:
@@ -6364,26 +6351,6 @@ bool X86InstrInfo::expandPostRAPseudo(MachineInstr &MI) const {
MIB->getOperand(0).setReg(SrcReg);
return Expand2AddrUndef(MIB, get(X86::VPXORDZrr));
}
- case X86::AVX512_256_SET0:
- case X86::AVX512_512_SET0: {
- bool HasVLX = Subtarget.hasVLX();
- Register SrcReg = MIB.getReg(0);
- const TargetRegisterInfo *TRI = &getRegisterInfo();
- if (HasVLX || TRI->getEncodingValue(SrcReg) < 16) {
- Register XReg = TRI->getSubReg(SrcReg, X86::sub_xmm);
- MIB->getOperand(0).setReg(XReg);
- Expand2AddrUndef(MIB, get(HasVLX ? X86::VPXORDZ128rr : X86::VXORPSrr));
- MIB.addReg(SrcReg, RegState::ImplicitDefine);
- return true;
- }
- if (MI.getOpcode() == X86::AVX512_256_SET0) {
- // No VLX so we must reference a zmm.
- MCRegister ZReg =
- TRI->getMatchingSuperReg(SrcReg, X86::sub_ymm, &X86::VR512RegClass);
- MIB->getOperand(0).setReg(ZReg);
- }
- return Expand2AddrUndef(MIB, get(X86::VPXORDZrr));
- }
case X86::MOVSHPmr:
case X86::MOVSHPrm:
return expandMOVSHP(MIB, MI, *this, Subtarget.hasAVX());
@@ -8486,14 +8453,11 @@ X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
Alignment = (*LoadMI.memoperands_begin())->getAlign();
else
switch (LoadOpc) {
- case X86::AVX512_512_SET0:
case X86::AVX512_512_SETALLONES:
Alignment = Align(64);
break;
case X86::AVX2_SETALLONES:
case X86::AVX1_SETALLONES:
- case X86::AVX_SET0:
- case X86::AVX512_256_SET0:
case X86::AVX512_256_SETALLONES:
Alignment = Align(32);
break;
@@ -8557,10 +8521,7 @@ X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
case X86::V_SETALLONES:
case X86::AVX2_SETALLONES:
case X86::AVX1_SETALLONES:
- case X86::AVX_SET0:
case X86::AVX512_128_SET0:
- case X86::AVX512_256_SET0:
- case X86::AVX512_512_SET0:
case X86::AVX512_128_SETALLONES:
case X86::AVX512_256_SETALLONES:
case X86::AVX512_512_SETALLONES:
@@ -8617,17 +8578,10 @@ X86InstrInfo::foldMemoryOperandImpl(MachineFunction &MF, MachineInstr &MI,
case X86::AVX512_512_SETALLONES:
IsAllOnes = true;
[[fallthrough]];
- case X86::AVX512_512_SET0:
- Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()),
- 16);
- break;
case X86::AVX1_SETALLONES:
case X86::AVX2_SETALLONES:
case X86::AVX512_256_SETALLONES:
IsAllOnes = true;
- [[fallthrough]];
- case X86::AVX512_256_SET0:
- case X86::AVX_SET0:
Ty = FixedVectorType::get(Type::getInt32Ty(MF.getFunction().getContext()),
8);
@@ -10905,13 +10859,14 @@ void X86InstrInfo::buildClearRegister(Register Reg, MachineBasicBlock &MBB,
if (!ST.hasAVX())
return;
- BuildMI(MBB, Iter, DL, get(X86::AVX_SET0), Reg);
+ BuildMI(MBB, Iter, DL, get(X86::V_SET0), TRI.getSubReg(Reg, X86::sub_xmm));
} else if (X86::VR512RegClass.contains(Reg)) {
// ZMM#
if (!ST.hasAVX512())
return;
- BuildMI(MBB, Iter, DL, get(X86::AVX512_512_SET0), Reg);
+ BuildMI(MBB, Iter, DL, get(X86::AVX512_128_SET0),
+ TRI.getSubReg(Reg, X86::sub_xmm));
} else if (X86::VK1RegClass.contains(Reg) || X86::VK2RegClass.contains(Reg) ||
X86::VK4RegClass.contains(Reg) || X86::VK8RegClass.contains(Reg) ||
X86::VK16RegClass.contains(Reg)) {
diff --git a/llvm/lib/Target/X86/X86InstrSSE.td b/llvm/lib/Target/X86/X86InstrSSE.td
index bf262315d7ea5..dd8c589c41589 100644
--- a/llvm/lib/Target/X86/X86InstrSSE.td
+++ b/llvm/lib/Target/X86/X86InstrSSE.td
@@ -146,24 +146,14 @@ def : Pat<(v2i64 immAllZerosV), (V_SET0)>;
def : Pat<(v2f64 immAllZerosV), (V_SET0)>;
}
-
-// The same as done above but for AVX. The 256-bit AVX1 ISA doesn't support PI,
-// and doesn't need it because on sandy bridge the register is set to zero
-// at the rename stage without using any execution unit, so SET0PSY
-// and SET0PDY can be used for vector int instructions without penalty
-let isReMaterializable = 1, isAsCheapAsAMove = 1, canFoldAsLoad = 1,
- isPseudo = 1, Predicates = [NoAVX512], SchedRW = [WriteZero] in {
-def AVX_SET0 : I<0, Pseudo, (outs VR256:$dst), (ins), "",
- [(set VR256:$dst, (v8i32 immAllZerosV))]>;
-}
-
let Predicates = [NoAVX512] in {
-def : Pat<(v32i8 immAllZerosV), (AVX_SET0)>;
-def : Pat<(v16i16 immAllZerosV), (AVX_SET0)>;
-def : Pat<(v16f16 immAllZerosV), (AVX_SET0)>;
-def : Pat<(v4i64 immAllZerosV), (AVX_SET0)>;
-def : Pat<(v8f32 immAllZerosV), (AVX_SET0)>;
-def : Pat<(v4f64 immAllZerosV), (AVX_SET0)>;
+def : Pat<(v32i8 immAllZerosV), (SUBREG_TO_REG (v4f32 (V_SET0)), sub_xmm)>;
+def : Pat<(v16i16 immAllZerosV), (SUBREG_TO_REG (v4f32 (V_SET0)), sub_xmm)>;
+def : Pat<(v16f16 immAllZerosV), (SUBREG_TO_REG (v4f32 (V_SET0)), sub_xmm)>;
+def : Pat<(v8i32 immAllZerosV), (SUBREG_TO_REG (v4f32 (V_SET0)), sub_xmm)>;
+def : Pat<(v8f32 immAllZerosV), (SUBREG_TO_REG (v4f32 (V_SET0)), sub_xmm)>;
+def : Pat<(v4i64 immAllZerosV), (SUBREG_TO_REG (v4f32 (V_SET0)), sub_xmm)>;
+def : Pat<(v4f64 immAllZerosV), (SUBREG_TO_REG (v4f32 (V_SET0)), sub_xmm)>;
}
// We set canFoldAsLoad because this can be converted to a constant-pool
@@ -7347,7 +7337,8 @@ def VINSERTF128rmi : AVXAIi8<0x18, MRMSrcMem, (outs VR256:$dst),
// with YMM register containing zero.
// FIXME: Avoid producing vxorps to clear the fake inputs.
let Predicates = [HasAVX1Only] in {
-def : Pat<(v8i32 immAllOnesV), (VCMPPSYrri (AVX_SET0), (AVX_SET0), 0xf)>;
+def : Pat<(v8i32 immAllOnesV), (VCMPPSYrri (SUBREG_TO_REG (v4f32 (V_SET0)), sub_xmm),
+ (SUBREG_TO_REG (v4f32 (V_SET0)), sub_xmm), 0xf)>;
}
multiclass vinsert_lowering<string InstrStr, string PermStr,
diff --git a/llvm/lib/Target/X86/X86PreTileConfig.cpp b/llvm/lib/Target/X86/X86PreTileConfig.cpp
index 4a3931f0f37fb..103b977ed7c84 100644
--- a/llvm/lib/Target/X86/X86PreTileConfig.cpp
+++ b/llvm/lib/Target/X86/X86PreTileConfig.cpp
@@ -417,13 +417,21 @@ bool X86PreTileConfigImpl::runOnMachineFunction(MachineFunction &MF) {
MachineBasicBlock &MBB = MF.front();
MachineInstr *MI = &*MBB.begin();
if (ST.hasAVX512()) {
+ Register Xmm = MRI->createVirtualRegister(&X86::VR128XRegClass);
+ BuildMI(MBB, MI, DL, TII->get(X86::AVX512_128_SET0), Xmm);
Register Zmm = MRI->createVirtualRegister(&X86::VR512RegClass);
- BuildMI(MBB, MI, DL, TII->get(X86::AVX512_512_SET0), Zmm);
+ BuildMI(MBB, MI, DL, TII->get(X86::SUBREG_TO_REG), Zmm)
+ .addReg(Xmm)
+ .addImm(X86::sub_xmm);
addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSZmr)), SS)
.addReg(Zmm);
} else if (ST.hasAVX2()) {
+ Register Xmm = MRI->createVirtualRegister(&X86::VR128RegClass);
+ BuildMI(MBB, MI, DL, TII->get(X86::V_SET0), Xmm);
Register Ymm = MRI->createVirtualRegister(&X86::VR256RegClass);
- BuildMI(MBB, MI, DL, TII->get(X86::AVX_SET0), Ymm);
+ BuildMI(MBB, MI, DL, TII->get(X86::SUBREG_TO_REG), Ymm)
+ .addReg(Xmm)
+ .addImm(X86::sub_xmm);
addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSYmr)), SS)
.addReg(Ymm);
addFrameReference(BuildMI(MBB, MI, DL, TII->get(X86::VMOVUPSYmr)), SS, 32)
diff --git a/llvm/test/CodeGen/MIR/X86/machine-verifier-address.mir b/llvm/test/CodeGen/MIR/X86/machine-verifier-address.mir
index 284e9288ca812..6399d8fd9ed75 100644
--- a/llvm/test/CodeGen/MIR/X86/machine-verifier-address.mir
+++ b/llvm/test/CodeGen/MIR/X86/machine-verifier-address.mir
@@ -16,12 +16,13 @@ body: |
bb.1:
successors: %bb.1(0x80000000)
- %3:vr256 = AVX_SET0
+ %3:vr128 = V_SET0
+ %5:vr256 = SUBREG_TO_REG %3, %subreg.sub_xmm
%4:vr128 = VPSLLDri %2, 31
- %5:vr256 = VPMOVSXDQYrr killed %4
+ %7:vr256 = VPMOVSXDQYrr killed %4
%8:vr256 = IMPLICIT_DEF
; CHECK: *** Bad machine code: Scale factor in address must be 1, 2, 4 or 8 ***
- %6:vr256, %7:vr256 = VGATHERQPDYrm %3, %0, 16, killed %8, 0, $noreg, %5 :: (load unknown-size, align 8)
+ %6:vr256, %7:vr256 = VGATHERQPDYrm %5, %0, 16, killed %8, 0, $noreg, %7 :: (load unknown-size, align 8)
%9:vr128 = COPY %6.sub_xmm
; CHECK: *** Bad machine code: Displacement in address must fit into 32-bit signed integer ***
VMOVLPDmr $noreg, 1, $noreg, 1111111111111, $noreg, killed %9 :: (store (s64) into `ptr poison`)
diff --git a/llvm/test/CodeGen/X86/2012-01-12-extract-sv.ll b/llvm/test/CodeGen/X86/2012-01-12-extract-sv.ll
index 65273870c3dfb..d7faf145049f8 100644
--- a/llvm/test/CodeGen/X86/2012-01-12-extract-sv.ll
+++ b/llvm/test/CodeGen/X86/2012-01-12-extract-sv.ll
@@ -10,7 +10,6 @@ define void @endless_loop() {
; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
; AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; AVX1-NEXT: vblendps {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5,6],ymm1[7]
-; AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; AVX1-NEXT: vmovss {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
; AVX1-NEXT: vmovaps %ymm0, (%eax)
; AVX1-NEXT: vmovaps %ymm1, (%eax)
@@ -22,8 +21,8 @@ define void @endless_loop() {
; AVX2-NEXT: vbroadcastss (%eax), %xmm0
; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vmovss {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3]
-; AVX2-NEXT: vbroadcastss %xmm0, %ymm0
; AVX2-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vbroadcastss %xmm0, %ymm0
; AVX2-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5,6],ymm0[7]
; AVX2-NEXT: vmovaps %ymm0, (%eax)
; AVX2-NEXT: vmovaps %ymm1, (%eax)
diff --git a/llvm/test/CodeGen/X86/AMX/amx-across-func.ll b/llvm/test/CodeGen/X86/AMX/amx-across-func.ll
index 1e752ef981960..cbd4ee0370642 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-across-func.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-across-func.ll
@@ -105,6 +105,7 @@ define dso_local void @test_api(i16 signext %0, i16 signext %1) nounwind {
; O0-NEXT: andq $-1024, %rsp # imm = 0xFC00
; O0-NEXT: subq $8192, %rsp # imm = 0x2000
; O0-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; O0-NEXT: # kill: def $zmm0 killed $xmm0
; O0-NEXT: vmovups %zmm0, {{[0-9]+}}(%rsp)
; O0-NEXT: movb $1, {{[0-9]+}}(%rsp)
; O0-NEXT: movw %si, %cx
@@ -338,6 +339,7 @@ define dso_local i32 @test_loop(i32 %0) nounwind {
; O0-NEXT: andq $-1024, %rsp # imm = 0xFC00
; O0-NEXT: subq $4096, %rsp # imm = 0x1000
; O0-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; O0-NEXT: # kill: def $zmm0 killed $xmm0
; O0-NEXT: vmovups %zmm0, {{[0-9]+}}(%rsp)
; O0-NEXT: movb $1, {{[0-9]+}}(%rsp)
; O0-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
@@ -557,6 +559,7 @@ define dso_local void @test_loop2(i32 %0) nounwind {
; O0-NEXT: andq $-1024, %rsp # imm = 0xFC00
; O0-NEXT: subq $3072, %rsp # imm = 0xC00
; O0-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; O0-NEXT: # kill: def $zmm0 killed $xmm0
; O0-NEXT: vmovups %zmm0, {{[0-9]+}}(%rsp)
; O0-NEXT: movb $1, {{[0-9]+}}(%rsp)
; O0-NEXT: movl %edi, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
diff --git a/llvm/test/CodeGen/X86/AMX/amx-configO0toO0.ll b/llvm/test/CodeGen/X86/AMX/amx-configO0toO0.ll
index 45f5f3b8dbb74..4ecb22f47e9af 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-configO0toO0.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-configO0toO0.ll
@@ -13,6 +13,7 @@ define dso_local void @test_api(i32 %cond, i16 signext %row, i16 signext %col) #
; AVX512-NEXT: andq $-1024, %rsp # imm = 0xFC00
; AVX512-NEXT: subq $25600, %rsp # imm = 0x6400
; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: # kill: def $zmm0 killed $xmm0
; AVX512-NEXT: vmovups %zmm0, {{[0-9]+}}(%rsp)
; AVX512-NEXT: movb $1, {{[0-9]+}}(%rsp)
; AVX512-NEXT: movw %dx, %ax
diff --git a/llvm/test/CodeGen/X86/AMX/amx-configO2toO0.ll b/llvm/test/CodeGen/X86/AMX/amx-configO2toO0.ll
index 3d4b4b88c82ad..5eb036ca47563 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-configO2toO0.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-configO2toO0.ll
@@ -12,6 +12,7 @@ define dso_local void @test_api(i32 %cond, i16 signext %row, i16 signext %col) n
; AVX512-NEXT: andq $-1024, %rsp # imm = 0xFC00
; AVX512-NEXT: subq $8192, %rsp # imm = 0x2000
; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: # kill: def $zmm0 killed $xmm0
; AVX512-NEXT: vmovups %zmm0, {{[0-9]+}}(%rsp)
; AVX512-NEXT: movb $1, {{[0-9]+}}(%rsp)
; AVX512-NEXT: movw %dx, %ax
diff --git a/llvm/test/CodeGen/X86/AMX/amx-fastconfig-spill.mir b/llvm/test/CodeGen/X86/AMX/amx-fastconfig-spill.mir
index f4870fc318d06..0d092ba38c757 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-fastconfig-spill.mir
+++ b/llvm/test/CodeGen/X86/AMX/amx-fastconfig-spill.mir
@@ -31,8 +31,9 @@ body: |
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[AVX512_512_SET0_:%[0-9]+]]:vr512 = AVX512_512_SET0
- ; CHECK-NEXT: VMOVUPSZmr %stack.4, 1, $noreg, 0, $noreg, [[AVX512_512_SET0_]] :: (store (s512) into %stack.4, align 4)
+ ; CHECK-NEXT: [[AVX512_128_SET0_:%[0-9]+]]:vr128x = AVX512_128_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:vr512 = SUBREG_TO_REG [[AVX512_128_SET0_]], %subreg.sub_xmm
+ ; CHECK-NEXT: VMOVUPSZmr %stack.4, 1, $noreg, 0, $noreg, [[SUBREG_TO_REG]] :: (store (s512) into %stack.4, align 4)
; CHECK-NEXT: MOV8mi %stack.4, 1, $noreg, 0, $noreg, 1 :: (store (s512) into %stack.4, align 4)
; CHECK-NEXT: [[MOV16ri:%[0-9]+]]:gr16 = MOV16ri 32
; CHECK-NEXT: [[MOV16ri1:%[0-9]+]]:gr16 = MOV16ri 8
@@ -109,8 +110,9 @@ body: |
; CHECK: bb.0.entry:
; CHECK-NEXT: successors: %bb.1(0x80000000)
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[AVX512_512_SET0_:%[0-9]+]]:vr512 = AVX512_512_SET0
- ; CHECK-NEXT: VMOVUPSZmr %stack.4, 1, $noreg, 0, $noreg, [[AVX512_512_SET0_]] :: (store (s512) into %stack.4, align 4)
+ ; CHECK-NEXT: [[AVX512_128_SET0_:%[0-9]+]]:vr128x = AVX512_128_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:vr512 = SUBREG_TO_REG [[AVX512_128_SET0_]], %subreg.sub_xmm
+ ; CHECK-NEXT: VMOVUPSZmr %stack.4, 1, $noreg, 0, $noreg, [[SUBREG_TO_REG]] :: (store (s512) into %stack.4, align 4)
; CHECK-NEXT: MOV8mi %stack.4, 1, $noreg, 0, $noreg, 1 :: (store (s512) into %stack.4, align 4)
; CHECK-NEXT: [[MOV16ri:%[0-9]+]]:gr16 = MOV16ri 32
; CHECK-NEXT: [[MOV16ri1:%[0-9]+]]:gr16 = MOV16ri 8
diff --git a/llvm/test/CodeGen/X86/AMX/amx-fastconfig.mir b/llvm/test/CodeGen/X86/AMX/amx-fastconfig.mir
index 8e0d320879344..ac2bcf3a54441 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-fastconfig.mir
+++ b/llvm/test/CodeGen/X86/AMX/amx-fastconfig.mir
@@ -86,8 +86,9 @@ body: |
; CHECK-NEXT: successors: %bb.2(0x30000000), %bb.1(0x50000000)
; CHECK-NEXT: liveins: $edi, $esi, $edx
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: [[AVX512_512_SET0_:%[0-9]+]]:vr512 = AVX512_512_SET0
- ; CHECK-NEXT: VMOVUPSZmr %stack.3, 1, $noreg, 0, $noreg, [[AVX512_512_SET0_]] :: (store (s512) into %stack.3, align 4)
+ ; CHECK-NEXT: [[AVX512_128_SET0_:%[0-9]+]]:vr128x = AVX512_128_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:vr512 = SUBREG_TO_REG [[AVX512_128_SET0_]], %subreg.sub_xmm
+ ; CHECK-NEXT: VMOVUPSZmr %stack.3, 1, $noreg, 0, $noreg, [[SUBREG_TO_REG]] :: (store (s512) into %stack.3, align 4)
; CHECK-NEXT: MOV8mi %stack.3, 1, $noreg, 0, $noreg, 1 :: (store (s512) into %stack.3, align 4)
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr32 = COPY killed $edx
; CHECK-NEXT: [[COPY1:%[0-9]+]]:gr32 = COPY killed $esi
diff --git a/llvm/test/CodeGen/X86/AMX/amx-fastpreconfig.mir b/llvm/test/CodeGen/X86/AMX/amx-fastpreconfig.mir
index 59678661c9409..803f85d06dcb5 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-fastpreconfig.mir
+++ b/llvm/test/CodeGen/X86/AMX/amx-fastpreconfig.mir
@@ -29,8 +29,9 @@ machineFunctionInfo:
body: |
bb.0.entry:
; CHECK-LABEL: name: main
- ; CHECK: [[AVX512_512_SET0_:%[0-9]+]]:vr512 = AVX512_512_SET0
- ; CHECK-NEXT: VMOVUPSZmr %stack.2, 1, $noreg, 0, $noreg, [[AVX512_512_SET0_]] :: (store (s512) into %stack.2, align 4)
+ ; CHECK: [[AVX512_128_SET0_:%[0-9]+]]:vr128x = AVX512_128_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:vr512 = SUBREG_TO_REG [[AVX512_128_SET0_]], %subreg.sub_xmm
+ ; CHECK-NEXT: VMOVUPSZmr %stack.2, 1, $noreg, 0, $noreg, [[SUBREG_TO_REG]] :: (store (s512) into %stack.2, align 4)
; CHECK-NEXT: MOV8mi %stack.2, 1, $noreg, 0, $noreg, 1 :: (store (s512) into %stack.2, align 4)
; CHECK-NEXT: [[MOV32ri64_:%[0-9]+]]:gr64_nosp = MOV32ri64 32
; CHECK-NEXT: [[LEA64r:%[0-9]+]]:gr64 = LEA64r %stack.0, 1, $noreg, 0, $noreg
diff --git a/llvm/test/CodeGen/X86/AMX/amx-greedy-ra-spill-shape.ll b/llvm/test/CodeGen/X86/AMX/amx-greedy-ra-spill-shape.ll
index 9a93b52f2d74c..0e2b6769f7010 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-greedy-ra-spill-shape.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-greedy-ra-spill-shape.ll
@@ -13,8 +13,8 @@ define void @foo(i32 %M, i32 %N, i32 %K, ptr %A, ptr %B_rcr4, ptr %C, i32 %c_row
; CHECK-NEXT: MOV64mr %stack.1, 1, $noreg, 0, $noreg, $rcx :: (store (s64) into %stack.1)
; CHECK-NEXT: undef [[COPY2:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = COPY $edx
; CHECK-NEXT: undef [[COPY3:%[0-9]+]].sub_32bit:gr64_nosp = COPY $esi
- ; CHECK-NEXT: [[AVX512_512_SET0_:%[0-9]+]]:vr512 = AVX512_512_SET0
- ; CHECK-NEXT: VMOVUPSZmr %stack.0, 1, $noreg, 0, $noreg, [[AVX512_512_SET0_]] :: (store (s512) into %stack.0, align 4)
+ ; CHECK-NEXT: undef [[AVX512_128_SET0_:%[0-9]+]].sub_xmm:vr512 = AVX512_128_SET0
+ ; CHECK-NEXT: VMOVUPSZmr %stack.0, 1, $noreg, 0, $noreg, [[AVX512_128_SET0_]] :: (store (s512) into %stack.0, align 4)
; CHECK-NEXT: MOV8mi %stack.0, 1, $noreg, 0, $noreg, 1 :: (store (s512) into %stack.0, align 4)
; CHECK-NEXT: [[MOV32rm:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.4, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.4, align 8)
; CHECK-NEXT: [[MOV32rm1:%[0-9]+]]:gr32 = MOV32rm %fixed-stack.5, 1, $noreg, 0, $noreg :: (load (s32) from %fixed-stack.5, align 16)
diff --git a/llvm/test/CodeGen/X86/AMX/amx-greedy-ra.ll b/llvm/test/CodeGen/X86/AMX/amx-greedy-ra.ll
index d258364536cfb..23a389d4b1202 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-greedy-ra.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-greedy-ra.ll
@@ -8,23 +8,23 @@ define i16 @foo(i32 noundef %t, i16 %row, i16 %col) nounwind {
; CHECK: bb.0.entry:
; CHECK-NEXT: liveins: $esi, $edx
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: undef %12.sub_32bit:gr64_nosp = COPY $edx
- ; CHECK-NEXT: undef %13.sub_32bit:gr64_with_sub_8bit = COPY $esi
- ; CHECK-NEXT: [[AVX512_512_SET0_:%[0-9]+]]:vr512 = AVX512_512_SET0
- ; CHECK-NEXT: VMOVUPSZmr %stack.0, 1, $noreg, 0, $noreg, [[AVX512_512_SET0_]] :: (store (s512) into %stack.0, align 4)
+ ; CHECK-NEXT: undef [[COPY:%[0-9]+]].sub_32bit:gr64_nosp = COPY $edx
+ ; CHECK-NEXT: undef [[COPY1:%[0-9]+]].sub_32bit:gr64_with_sub_8bit = COPY $esi
+ ; CHECK-NEXT: undef [[AVX512_128_SET0_:%[0-9]+]].sub_xmm:vr512 = AVX512_128_SET0
+ ; CHECK-NEXT: VMOVUPSZmr %stack.0, 1, $noreg, 0, $noreg, [[AVX512_128_SET0_]] :: (store (s512) into %stack.0, align 4)
; CHECK-NEXT: MOV8mi %stack.0, 1, $noreg, 0, $noreg, 1 :: (store (s512) into %stack.0, align 4)
- ; CHECK-NEXT: MOV16mr %stack.0, 1, $noreg, 20, $noreg, %12.sub_16bit :: (store (s512) into %stack.0 + 20, align 4)
- ; CHECK-NEXT: MOV8mr %stack.0, 1, $noreg, 50, $noreg, %13.sub_8bit :: (store (s512) into %stack.0 + 50, align 2, basealign 4)
- ; CHECK-NEXT: MOV16mr %stack.0, 1, $noreg, 18, $noreg, %12.sub_16bit :: (store (s512) into %stack.0 + 18, align 2, basealign 4)
- ; CHECK-NEXT: MOV8mr %stack.0, 1, $noreg, 49, $noreg, %13.sub_8bit :: (store (s512) into %stack.0 + 49, align 1, basealign 4)
- ; CHECK-NEXT: MOV16mr %stack.0, 1, $noreg, 16, $noreg, %12.sub_16bit :: (store (s512) into %stack.0 + 16, align 4)
- ; CHECK-NEXT: MOV8mr %stack.0, 1, $noreg, 48, $noreg, %13.sub_8bit :: (store (s512) into %stack.0 + 48, align 4)
+ ; CHECK-NEXT: MOV16mr %stack.0, 1, $noreg, 20, $noreg, [[COPY]].sub_16bit :: (store (s512) into %stack.0 + 20, align 4)
+ ; CHECK-NEXT: MOV8mr %stack.0, 1, $noreg, 50, $noreg, [[COPY1]].sub_8bit :: (store (s512) into %stack.0 + 50, align 2, basealign 4)
+ ; CHECK-NEXT: MOV16mr %stack.0, 1, $noreg, 18, $noreg, [[COPY]].sub_16bit :: (store (s512) into %stack.0 + 18, align 2, basealign 4)
+ ; CHECK-NEXT: MOV8mr %stack.0, 1, $noreg, 49, $noreg, [[COPY1]].sub_8bit :: (store (s512) into %stack.0 + 49, align 1, basealign 4)
+ ; CHECK-NEXT: MOV16mr %stack.0, 1, $noreg, 16, $noreg, [[COPY]].sub_16bit :: (store (s512) into %stack.0 + 16, align 4)
+ ; CHECK-NEXT: MOV8mr %stack.0, 1, $noreg, 48, $noreg, [[COPY1]].sub_8bit :: (store (s512) into %stack.0 + 48, align 4)
; CHECK-NEXT: PLDTILECFGV %stack.0, 1, $noreg, 0, $noreg, implicit-def dead $tmm0, implicit-def dead $tmm1, implicit-def dead $tmm2, implicit-def dead $tmm3, implicit-def dead $tmm4, implicit-def dead $tmm5, implicit-def dead $tmm6, implicit-def dead $tmm7 :: (load (s512) from %stack.0, align 4)
- ; CHECK-NEXT: [[PTILEZEROV:%[0-9]+]]:tile = PTILEZEROV %13.sub_16bit, %12.sub_16bit
- ; CHECK-NEXT: [[PTILEZEROV1:%[0-9]+]]:tile = PTILEZEROV %13.sub_16bit, %12.sub_16bit
- ; CHECK-NEXT: [[PTILEZEROV2:%[0-9]+]]:tile = PTILEZEROV %13.sub_16bit, %12.sub_16bit
- ; CHECK-NEXT: dead [[PTILEZEROV2]]:tile = PTDPBSSDV %13.sub_16bit, %12.sub_16bit, %12.sub_16bit, [[PTILEZEROV2]], [[PTILEZEROV]], [[PTILEZEROV1]]
- ; CHECK-NEXT: [[LEA64_32r:%[0-9]+]]:gr32 = LEA64_32r %13, 1, %12, 0, $noreg
+ ; CHECK-NEXT: [[PTILEZEROV:%[0-9]+]]:tile = PTILEZEROV [[COPY1]].sub_16bit, [[COPY]].sub_16bit
+ ; CHECK-NEXT: [[PTILEZEROV1:%[0-9]+]]:tile = PTILEZEROV [[COPY1]].sub_16bit, [[COPY]].sub_16bit
+ ; CHECK-NEXT: [[PTILEZEROV2:%[0-9]+]]:tile = PTILEZEROV [[COPY1]].sub_16bit, [[COPY]].sub_16bit
+ ; CHECK-NEXT: dead [[PTILEZEROV2:%[0-9]+]]:tile = PTDPBSSDV [[COPY1]].sub_16bit, [[COPY]].sub_16bit, [[COPY]].sub_16bit, [[PTILEZEROV2]], [[PTILEZEROV]], [[PTILEZEROV1]]
+ ; CHECK-NEXT: [[LEA64_32r:%[0-9]+]]:gr32 = LEA64_32r [[COPY1]], 1, [[COPY]], 0, $noreg
; CHECK-NEXT: $ax = COPY [[LEA64_32r]].sub_16bit
; CHECK-NEXT: RET 0, killed $ax
entry:
diff --git a/llvm/test/CodeGen/X86/AMX/amx-sink-config-after-calls.mir b/llvm/test/CodeGen/X86/AMX/amx-sink-config-after-calls.mir
index 320118cc19376..88f1a52a1f9c4 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-sink-config-after-calls.mir
+++ b/llvm/test/CodeGen/X86/AMX/amx-sink-config-after-calls.mir
@@ -61,7 +61,8 @@ body: |
; CHECK-LABEL: name: test_api
; CHECK: liveins: $rdi, $rsi
; CHECK-NEXT: {{ $}}
- ; CHECK-NEXT: renamable $zmm0 = AVX512_512_SET0
+ ; CHECK-NEXT: renamable $xmm0 = AVX512_128_SET0
+ ; CHECK-NEXT: renamable $zmm0 = SUBREG_TO_REG killed renamable $xmm0, %subreg.sub_xmm
; CHECK-NEXT: VMOVUPSZmr %stack.5, 1, $noreg, 0, $noreg, killed renamable $zmm0 :: (store (s512) into %stack.5, align 4)
; CHECK-NEXT: MOV8mi %stack.5, 1, $noreg, 0, $noreg, 1 :: (store (s512) into %stack.5, align 4)
; CHECK-NEXT: MOV64mr %stack.8, 1, $noreg, 0, $noreg, $rsi :: (store (s64) into %stack.8)
diff --git a/llvm/test/CodeGen/X86/AMX/amx-zero-config.ll b/llvm/test/CodeGen/X86/AMX/amx-zero-config.ll
index 1eb641b3d73e5..c4901ab136c71 100644
--- a/llvm/test/CodeGen/X86/AMX/amx-zero-config.ll
+++ b/llvm/test/CodeGen/X86/AMX/amx-zero-config.ll
@@ -68,6 +68,7 @@ define void @foo(ptr %buf) nounwind {
; AVX512-O0-NEXT: andq $-1024, %rsp # imm = 0xFC00
; AVX512-O0-NEXT: subq $3072, %rsp # imm = 0xC00
; AVX512-O0-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512-O0-NEXT: # kill: def $zmm0 killed $xmm0
; AVX512-O0-NEXT: vmovups %zmm0, {{[0-9]+}}(%rsp)
; AVX512-O0-NEXT: movb $1, {{[0-9]+}}(%rsp)
; AVX512-O0-NEXT: movw $32, %cx
@@ -108,6 +109,7 @@ define void @foo(ptr %buf) nounwind {
; AVX2-O0-NEXT: andq $-1024, %rsp # imm = 0xFC00
; AVX2-O0-NEXT: subq $3072, %rsp # imm = 0xC00
; AVX2-O0-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-O0-NEXT: # kill: def $ymm0 killed $xmm0
; AVX2-O0-NEXT: vmovups %ymm0, {{[0-9]+}}(%rsp)
; AVX2-O0-NEXT: vmovups %ymm0, {{[0-9]+}}(%rsp)
; AVX2-O0-NEXT: movb $1, {{[0-9]+}}(%rsp)
diff --git a/llvm/test/CodeGen/X86/amx-across-func-tilemovrow.ll b/llvm/test/CodeGen/X86/amx-across-func-tilemovrow.ll
index 885bc805d6552..d317deadb0354 100644
--- a/llvm/test/CodeGen/X86/amx-across-func-tilemovrow.ll
+++ b/llvm/test/CodeGen/X86/amx-across-func-tilemovrow.ll
@@ -96,6 +96,7 @@ define dso_local <16 x i32> @test_api(i16 signext %0, i16 signext %1) nounwind {
; O0-NEXT: andq $-1024, %rsp # imm = 0xFC00
; O0-NEXT: subq $4096, %rsp # imm = 0x1000
; O0-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; O0-NEXT: # kill: def $zmm0 killed $xmm0
; O0-NEXT: vmovups %zmm0, {{[0-9]+}}(%rsp)
; O0-NEXT: movb $1, {{[0-9]+}}(%rsp)
; O0-NEXT: movw %si, %cx
diff --git a/llvm/test/CodeGen/X86/apx/kmov-isel.ll b/llvm/test/CodeGen/X86/apx/kmov-isel.ll
index 882269b498261..39d8adc9661fb 100644
--- a/llvm/test/CodeGen/X86/apx/kmov-isel.ll
+++ b/llvm/test/CodeGen/X86/apx/kmov-isel.ll
@@ -68,8 +68,8 @@ define <32 x i1> @bitcast_32i8_load(ptr %p, <32 x i1> %a, <32 x i1> %b) {
; AVX512: # %bb.0:
; AVX512-NEXT: vpsllw $7, %ymm1, %ymm1 # encoding: [0xc5,0xf5,0x71,0xf1,0x07]
; AVX512-NEXT: vpmovb2m %zmm1, %k0 # encoding: [0x62,0xf2,0x7e,0x48,0x29,0xc1]
-; AVX512-NEXT: vpsllw $7, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x71,0xf0,0x07]
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1 # encoding: [0xc5,0xf1,0xef,0xc9]
+; AVX512-NEXT: vpsllw $7, %ymm0, %ymm0 # encoding: [0xc5,0xfd,0x71,0xf0,0x07]
; AVX512-NEXT: kmovd (%rdi), %k1 # EVEX TO VEX Compression encoding: [0xc4,0xe1,0xf9,0x90,0x0f]
; AVX512-NEXT: vpcmpgtb %zmm0, %zmm1, %k2 {%k1} # encoding: [0x62,0xf1,0x75,0x49,0x64,0xd0]
; AVX512-NEXT: kandnd %k0, %k1, %k0 # encoding: [0xc4,0xe1,0xf5,0x42,0xc0]
diff --git a/llvm/test/CodeGen/X86/avx-select.ll b/llvm/test/CodeGen/X86/avx-select.ll
index 1b688c8cf9cca..dab04c5f3bd26 100644
--- a/llvm/test/CodeGen/X86/avx-select.ll
+++ b/llvm/test/CodeGen/X86/avx-select.ll
@@ -6,22 +6,24 @@ define <8 x i32> @select00(i32 %a, <8 x i32> %b) nounwind {
; X86-LABEL: select00:
; X86: # %bb.0:
; X86-NEXT: cmpl $255, {{[0-9]+}}(%esp)
+; X86-NEXT: je .LBB0_1
+; X86-NEXT: # %bb.2:
+; X86-NEXT: vxorps %ymm0, %ymm0, %ymm0
+; X86-NEXT: retl
+; X86-NEXT: .LBB0_1:
; X86-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X86-NEXT: je .LBB0_2
-; X86-NEXT: # %bb.1:
-; X86-NEXT: vmovaps %ymm0, %ymm1
-; X86-NEXT: .LBB0_2:
; X86-NEXT: vxorps %ymm1, %ymm0, %ymm0
; X86-NEXT: retl
;
; X64-LABEL: select00:
; X64: # %bb.0:
; X64-NEXT: cmpl $255, %edi
+; X64-NEXT: je .LBB0_1
+; X64-NEXT: # %bb.2:
+; X64-NEXT: vxorps %ymm0, %ymm0, %ymm0
+; X64-NEXT: retq
+; X64-NEXT: .LBB0_1:
; X64-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64-NEXT: je .LBB0_2
-; X64-NEXT: # %bb.1:
-; X64-NEXT: vmovaps %ymm0, %ymm1
-; X64-NEXT: .LBB0_2:
; X64-NEXT: vxorps %ymm1, %ymm0, %ymm0
; X64-NEXT: retq
%cmpres = icmp eq i32 %a, 255
@@ -34,22 +36,24 @@ define <4 x i64> @select01(i32 %a, <4 x i64> %b) nounwind {
; X86-LABEL: select01:
; X86: # %bb.0:
; X86-NEXT: cmpl $255, {{[0-9]+}}(%esp)
+; X86-NEXT: je .LBB1_1
+; X86-NEXT: # %bb.2:
+; X86-NEXT: vxorps %ymm0, %ymm0, %ymm0
+; X86-NEXT: retl
+; X86-NEXT: .LBB1_1:
; X86-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X86-NEXT: je .LBB1_2
-; X86-NEXT: # %bb.1:
-; X86-NEXT: vmovaps %ymm0, %ymm1
-; X86-NEXT: .LBB1_2:
; X86-NEXT: vxorps %ymm1, %ymm0, %ymm0
; X86-NEXT: retl
;
; X64-LABEL: select01:
; X64: # %bb.0:
; X64-NEXT: cmpl $255, %edi
+; X64-NEXT: je .LBB1_1
+; X64-NEXT: # %bb.2:
+; X64-NEXT: vxorps %ymm0, %ymm0, %ymm0
+; X64-NEXT: retq
+; X64-NEXT: .LBB1_1:
; X64-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64-NEXT: je .LBB1_2
-; X64-NEXT: # %bb.1:
-; X64-NEXT: vmovaps %ymm0, %ymm1
-; X64-NEXT: .LBB1_2:
; X64-NEXT: vxorps %ymm1, %ymm0, %ymm0
; X64-NEXT: retq
%cmpres = icmp eq i32 %a, 255
diff --git a/llvm/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll
index a93ac67905979..818065aaa57d8 100644
--- a/llvm/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx2-intrinsics-fast-isel.ll
@@ -881,16 +881,16 @@ define <4 x i64> @test_mm256_i32gather_epi64(ptr%a0, <2 x i64> %a1) {
; X86-LABEL: test_mm256_i32gather_epi64:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; X86-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; X86-NEXT: vpgatherdq %ymm2, (%eax,%xmm0,2), %ymm1
; X86-NEXT: vmovdqa %ymm1, %ymm0
; X86-NEXT: retl
;
; X64-LABEL: test_mm256_i32gather_epi64:
; X64: # %bb.0:
-; X64-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; X64-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; X64-NEXT: vpgatherdq %ymm2, (%rdi,%xmm0,2), %ymm1
; X64-NEXT: vmovdqa %ymm1, %ymm0
; X64-NEXT: retq
@@ -968,6 +968,7 @@ define <4 x double> @test_mm256_i32gather_pd(ptr%a0, <2 x i64> %a1) {
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: vxorpd %xmm1, %xmm1, %xmm1
; X86-NEXT: vcmpeqpd %ymm1, %ymm1, %ymm2
+; X86-NEXT: vxorpd %xmm1, %xmm1, %xmm1
; X86-NEXT: vgatherdpd %ymm2, (%eax,%xmm0,2), %ymm1
; X86-NEXT: vmovapd %ymm1, %ymm0
; X86-NEXT: retl
@@ -976,6 +977,7 @@ define <4 x double> @test_mm256_i32gather_pd(ptr%a0, <2 x i64> %a1) {
; X64: # %bb.0:
; X64-NEXT: vxorpd %xmm1, %xmm1, %xmm1
; X64-NEXT: vcmpeqpd %ymm1, %ymm1, %ymm2
+; X64-NEXT: vxorpd %xmm1, %xmm1, %xmm1
; X64-NEXT: vgatherdpd %ymm2, (%rdi,%xmm0,2), %ymm1
; X64-NEXT: vmovapd %ymm1, %ymm0
; X64-NEXT: retq
@@ -1054,6 +1056,7 @@ define <8 x float> @test_mm256_i32gather_ps(ptr%a0, <4 x i64> %a1) {
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X86-NEXT: vcmpeqps %ymm1, %ymm1, %ymm2
+; X86-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X86-NEXT: vgatherdps %ymm2, (%eax,%ymm0,2), %ymm1
; X86-NEXT: vmovaps %ymm1, %ymm0
; X86-NEXT: retl
@@ -1062,6 +1065,7 @@ define <8 x float> @test_mm256_i32gather_ps(ptr%a0, <4 x i64> %a1) {
; X64: # %bb.0:
; X64-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X64-NEXT: vcmpeqps %ymm1, %ymm1, %ymm2
+; X64-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X64-NEXT: vgatherdps %ymm2, (%rdi,%ymm0,2), %ymm1
; X64-NEXT: vmovaps %ymm1, %ymm0
; X64-NEXT: retq
@@ -1225,16 +1229,16 @@ define <4 x i64> @test_mm256_i64gather_epi64(ptr%a0, <4 x i64> %a1) {
; X86-LABEL: test_mm256_i64gather_epi64:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; X86-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; X86-NEXT: vpgatherqq %ymm2, (%eax,%ymm0,2), %ymm1
; X86-NEXT: vmovdqa %ymm1, %ymm0
; X86-NEXT: retl
;
; X64-LABEL: test_mm256_i64gather_epi64:
; X64: # %bb.0:
-; X64-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; X64-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; X64-NEXT: vpgatherqq %ymm2, (%rdi,%ymm0,2), %ymm1
; X64-NEXT: vmovdqa %ymm1, %ymm0
; X64-NEXT: retq
@@ -1308,6 +1312,7 @@ define <4 x double> @test_mm256_i64gather_pd(ptr%a0, <4 x i64> %a1) {
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: vxorpd %xmm1, %xmm1, %xmm1
; X86-NEXT: vcmpeqpd %ymm1, %ymm1, %ymm2
+; X86-NEXT: vxorpd %xmm1, %xmm1, %xmm1
; X86-NEXT: vgatherqpd %ymm2, (%eax,%ymm0,2), %ymm1
; X86-NEXT: vmovapd %ymm1, %ymm0
; X86-NEXT: retl
@@ -1316,6 +1321,7 @@ define <4 x double> @test_mm256_i64gather_pd(ptr%a0, <4 x i64> %a1) {
; X64: # %bb.0:
; X64-NEXT: vxorpd %xmm1, %xmm1, %xmm1
; X64-NEXT: vcmpeqpd %ymm1, %ymm1, %ymm2
+; X64-NEXT: vxorpd %xmm1, %xmm1, %xmm1
; X64-NEXT: vgatherqpd %ymm2, (%rdi,%ymm0,2), %ymm1
; X64-NEXT: vmovapd %ymm1, %ymm0
; X64-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/avx512-build-vector.ll b/llvm/test/CodeGen/X86/avx512-build-vector.ll
index 55478a2e93154..1c51b091b2006 100644
--- a/llvm/test/CodeGen/X86/avx512-build-vector.ll
+++ b/llvm/test/CodeGen/X86/avx512-build-vector.ll
@@ -15,9 +15,9 @@ define <16 x float> @test3(<4 x float> %a) {
; CHECK-LABEL: test3:
; CHECK: ## %bb.0:
; CHECK-NEXT: ## kill: def $xmm0 killed $xmm0 def $zmm0
-; CHECK-NEXT: vmovaps {{.*#+}} zmm2 = [0,1,2,3,4,18,16,7,8,9,10,11,12,13,14,15]
-; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT: vpermt2ps %zmm0, %zmm2, %zmm1
+; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vmovaps {{.*#+}} zmm1 = [0,1,2,3,4,18,16,7,8,9,10,11,12,13,14,15]
+; CHECK-NEXT: vpermi2ps %zmm0, %zmm2, %zmm1
; CHECK-NEXT: vmovaps %zmm1, %zmm0
; CHECK-NEXT: retq
%b = extractelement <4 x float> %a, i32 2
diff --git a/llvm/test/CodeGen/X86/avx512-gather-scatter-intrin-deprecated.ll b/llvm/test/CodeGen/X86/avx512-gather-scatter-intrin-deprecated.ll
index f807a739b95fd..13ee74b759834 100644
--- a/llvm/test/CodeGen/X86/avx512-gather-scatter-intrin-deprecated.ll
+++ b/llvm/test/CodeGen/X86/avx512-gather-scatter-intrin-deprecated.ll
@@ -520,8 +520,8 @@ define <8 x float>@test_int_x86_avx512_gather3siv8_sf(<8 x float> %x0, ptr %x1,
; CHECK: ## %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vgatherdps (%rdi,%ymm1,4), %ymm0 {%k1}
-; CHECK-NEXT: kxnorb %k0, %k0, %k1
; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: kxnorb %k0, %k0, %k1
; CHECK-NEXT: vgatherdps (%rdi,%ymm1,2), %ymm2 {%k1}
; CHECK-NEXT: vaddps %ymm2, %ymm0, %ymm0
; CHECK-NEXT: retq
@@ -822,8 +822,8 @@ define void @scatter_mask_test(ptr %x0, <8 x i32> %x2, <8 x i32> %x3) {
define <16 x float> @gather_mask_test(<16 x i32> %ind, <16 x float> %src, ptr %base) {
; CHECK-LABEL: gather_mask_test:
; CHECK: ## %bb.0:
-; CHECK-NEXT: kxnorw %k0, %k0, %k1
; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: kxnorw %k0, %k0, %k1
; CHECK-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm2 {%k1}
; CHECK-NEXT: kxorw %k0, %k0, %k1
; CHECK-NEXT: vmovaps %zmm1, %zmm3
diff --git a/llvm/test/CodeGen/X86/avx512-gather-scatter-intrin.ll b/llvm/test/CodeGen/X86/avx512-gather-scatter-intrin.ll
index 58f33c49f88ee..20d3e4b3d052e 100644
--- a/llvm/test/CodeGen/X86/avx512-gather-scatter-intrin.ll
+++ b/llvm/test/CodeGen/X86/avx512-gather-scatter-intrin.ll
@@ -251,8 +251,8 @@ define dso_local void @scatter_mask_qps_execdomain(<8 x i64> %ind, ptr %src, i8
define dso_local void @gather_qps(<8 x i64> %ind, <8 x float> %src, ptr %base, ptr %stbuf) {
; CHECK-LABEL: gather_qps:
; CHECK: # %bb.0:
-; CHECK-NEXT: kxnorb %k0, %k0, %k1
; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: kxnorb %k0, %k0, %k1
; CHECK-NEXT: kxnorb %k0, %k0, %k2
; CHECK-NEXT: vgatherqps (%rdi,%zmm0,4), %ymm1 {%k2}
; CHECK-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0 # [0,1,2,3,0,1,2,3]
@@ -523,8 +523,8 @@ define <8 x float> @test_int_x86_avx512_mask_gather3siv8_sf(<8 x float> %x0, ptr
; CHECK: # %bb.0:
; CHECK-NEXT: kmovd %esi, %k1
; CHECK-NEXT: vgatherdps (%rdi,%ymm1,4), %ymm0 {%k1}
-; CHECK-NEXT: kxnorb %k0, %k0, %k1
; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: kxnorb %k0, %k0, %k1
; CHECK-NEXT: vgatherdps (%rdi,%ymm1,2), %ymm2 {%k1}
; CHECK-NEXT: vaddps %ymm2, %ymm0, %ymm0
; CHECK-NEXT: retq
@@ -824,8 +824,8 @@ define dso_local void @scatter_mask_test(ptr %x0, <8 x i32> %x2, <8 x i32> %x3)
define <16 x float> @gather_mask_test(<16 x i32> %ind, <16 x float> %src, ptr %base) {
; CHECK-LABEL: gather_mask_test:
; CHECK: # %bb.0:
-; CHECK-NEXT: kxnorw %k0, %k0, %k1
; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: kxnorw %k0, %k0, %k1
; CHECK-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm2 {%k1}
; CHECK-NEXT: kxorw %k0, %k0, %k1
; CHECK-NEXT: vmovaps %zmm1, %zmm3
@@ -856,8 +856,8 @@ define <16 x float> @gather_mask_test(<16 x i32> %ind, <16 x float> %src, ptr %b
define <8 x float> @gather_global(<8 x i64>, ptr nocapture readnone) {
; CHECK-LABEL: gather_global:
; CHECK: # %bb.0:
-; CHECK-NEXT: kxnorb %k0, %k0, %k1
; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: kxnorb %k0, %k0, %k1
; CHECK-NEXT: vgatherqps x(,%zmm0,4), %ymm1 {%k1}
; CHECK-NEXT: vmovaps %ymm1, %ymm0
; CHECK-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/avx512-insert-extract.ll b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
index 4b44afb33d103..3b151401ff67e 100644
--- a/llvm/test/CodeGen/X86/avx512-insert-extract.ll
+++ b/llvm/test/CodeGen/X86/avx512-insert-extract.ll
@@ -1665,11 +1665,11 @@ define i64 @test_insertelement_variable_v64i1(<64 x i8> %a, i8 %b, i32 %index) n
; KNL-NEXT: andq $-64, %rsp
; KNL-NEXT: subq $128, %rsp
; KNL-NEXT: ## kill: def $esi killed $esi def $rsi
-; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; KNL-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; KNL-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm1
-; KNL-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm0
-; KNL-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; KNL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; KNL-NEXT: vpcmpeqb %ymm1, %ymm2, %ymm2
+; KNL-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm0
+; KNL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; KNL-NEXT: vpternlogq {{.*#+}} zmm0 = ~zmm0
; KNL-NEXT: andl $63, %esi
; KNL-NEXT: testb %dil, %dil
@@ -2033,16 +2033,16 @@ define i128 @test_insertelement_variable_v128i1(<128 x i8> %a, i8 %b, i32 %index
; KNL-NEXT: andq $-64, %rsp
; KNL-NEXT: subq $192, %rsp
; KNL-NEXT: ## kill: def $esi killed $esi def $rsi
-; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; KNL-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; KNL-NEXT: vpcmpeqb %ymm3, %ymm2, %ymm2
-; KNL-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm0
-; KNL-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
+; KNL-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; KNL-NEXT: vpcmpeqb %ymm2, %ymm3, %ymm3
+; KNL-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm0
+; KNL-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
; KNL-NEXT: vpternlogq {{.*#+}} zmm0 = ~zmm0
-; KNL-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; KNL-NEXT: vpcmpeqb %ymm3, %ymm2, %ymm2
-; KNL-NEXT: vpcmpeqb %ymm3, %ymm1, %ymm1
-; KNL-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; KNL-NEXT: vextracti64x4 $1, %zmm1, %ymm3
+; KNL-NEXT: vpcmpeqb %ymm2, %ymm3, %ymm3
+; KNL-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm1
+; KNL-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
; KNL-NEXT: vpternlogq {{.*#+}} zmm1 = ~zmm1
; KNL-NEXT: andl $127, %esi
; KNL-NEXT: testb %dil, %dil
diff --git a/llvm/test/CodeGen/X86/avx512-mask-op.ll b/llvm/test/CodeGen/X86/avx512-mask-op.ll
index db53dda401bc2..f42afb4c031a3 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-op.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-op.ll
@@ -4092,23 +4092,23 @@ exit:
define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z) {
; KNL-LABEL: ktest_6:
; KNL: ## %bb.0:
-; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm4
-; KNL-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; KNL-NEXT: vpcmpeqw %ymm5, %ymm4, %ymm4
-; KNL-NEXT: vpcmpeqw %ymm5, %ymm0, %ymm0
-; KNL-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm0
-; KNL-NEXT: vextracti64x4 $1, %zmm1, %ymm4
-; KNL-NEXT: vpcmpeqw %ymm5, %ymm4, %ymm4
-; KNL-NEXT: vpcmpeqw %ymm5, %ymm1, %ymm1
-; KNL-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm1
+; KNL-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm5
+; KNL-NEXT: vpcmpeqw %ymm4, %ymm5, %ymm5
+; KNL-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0
+; KNL-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm0
+; KNL-NEXT: vextracti64x4 $1, %zmm1, %ymm5
+; KNL-NEXT: vpcmpeqw %ymm4, %ymm5, %ymm5
+; KNL-NEXT: vpcmpeqw %ymm4, %ymm1, %ymm1
+; KNL-NEXT: vinserti64x4 $1, %ymm5, %zmm1, %zmm1
; KNL-NEXT: vporq %zmm1, %zmm0, %zmm0
; KNL-NEXT: vextracti64x4 $1, %zmm2, %ymm1
-; KNL-NEXT: vpcmpeqw %ymm5, %ymm1, %ymm1
-; KNL-NEXT: vpcmpeqw %ymm5, %ymm2, %ymm2
+; KNL-NEXT: vpcmpeqw %ymm4, %ymm1, %ymm1
+; KNL-NEXT: vpcmpeqw %ymm4, %ymm2, %ymm2
; KNL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; KNL-NEXT: vextracti64x4 $1, %zmm3, %ymm2
-; KNL-NEXT: vpcmpeqw %ymm5, %ymm2, %ymm2
-; KNL-NEXT: vpcmpeqw %ymm5, %ymm3, %ymm3
+; KNL-NEXT: vpcmpeqw %ymm4, %ymm2, %ymm2
+; KNL-NEXT: vpcmpeqw %ymm4, %ymm3, %ymm3
; KNL-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2
; KNL-NEXT: vpternlogq {{.*#+}} zmm2 = zmm0 & (zmm2 | zmm1)
; KNL-NEXT: vextracti64x4 $1, %zmm2, %ymm0
@@ -4173,23 +4173,23 @@ define void @ktest_6(<32 x i16> %w, <32 x i16> %x, <32 x i16> %y, <32 x i16> %z)
;
; AVX512DQ-LABEL: ktest_6:
; AVX512DQ: ## %bb.0:
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm4
-; AVX512DQ-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX512DQ-NEXT: vpcmpeqw %ymm5, %ymm4, %ymm4
-; AVX512DQ-NEXT: vpcmpeqw %ymm5, %ymm0, %ymm0
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm0
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm4
-; AVX512DQ-NEXT: vpcmpeqw %ymm5, %ymm4, %ymm4
-; AVX512DQ-NEXT: vpcmpeqw %ymm5, %ymm1, %ymm1
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm5
+; AVX512DQ-NEXT: vpcmpeqw %ymm4, %ymm5, %ymm5
+; AVX512DQ-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm0
+; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm5
+; AVX512DQ-NEXT: vpcmpeqw %ymm4, %ymm5, %ymm5
+; AVX512DQ-NEXT: vpcmpeqw %ymm4, %ymm1, %ymm1
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm1, %zmm1
; AVX512DQ-NEXT: vporq %zmm1, %zmm0, %zmm0
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm2, %ymm1
-; AVX512DQ-NEXT: vpcmpeqw %ymm5, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpcmpeqw %ymm5, %ymm2, %ymm2
+; AVX512DQ-NEXT: vpcmpeqw %ymm4, %ymm1, %ymm1
+; AVX512DQ-NEXT: vpcmpeqw %ymm4, %ymm2, %ymm2
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm3, %ymm2
-; AVX512DQ-NEXT: vpcmpeqw %ymm5, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpcmpeqw %ymm5, %ymm3, %ymm3
+; AVX512DQ-NEXT: vpcmpeqw %ymm4, %ymm2, %ymm2
+; AVX512DQ-NEXT: vpcmpeqw %ymm4, %ymm3, %ymm3
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm2 = zmm0 & (zmm2 | zmm1)
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm2, %ymm0
@@ -4252,23 +4252,23 @@ exit:
define void @ktest_7(<64 x i8> %w, <64 x i8> %x, <64 x i8> %y, <64 x i8> %z) {
; KNL-LABEL: ktest_7:
; KNL: ## %bb.0:
-; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm4
-; KNL-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; KNL-NEXT: vpcmpeqb %ymm5, %ymm4, %ymm4
-; KNL-NEXT: vpcmpeqb %ymm5, %ymm0, %ymm0
-; KNL-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm0
-; KNL-NEXT: vextracti64x4 $1, %zmm1, %ymm4
-; KNL-NEXT: vpcmpeqb %ymm5, %ymm4, %ymm4
-; KNL-NEXT: vpcmpeqb %ymm5, %ymm1, %ymm1
-; KNL-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm1
+; KNL-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm5
+; KNL-NEXT: vpcmpeqb %ymm4, %ymm5, %ymm5
+; KNL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0
+; KNL-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm0
+; KNL-NEXT: vextracti64x4 $1, %zmm1, %ymm5
+; KNL-NEXT: vpcmpeqb %ymm4, %ymm5, %ymm5
+; KNL-NEXT: vpcmpeqb %ymm4, %ymm1, %ymm1
+; KNL-NEXT: vinserti64x4 $1, %ymm5, %zmm1, %zmm1
; KNL-NEXT: vporq %zmm1, %zmm0, %zmm0
; KNL-NEXT: vextracti64x4 $1, %zmm2, %ymm1
-; KNL-NEXT: vpcmpeqb %ymm5, %ymm1, %ymm1
-; KNL-NEXT: vpcmpeqb %ymm5, %ymm2, %ymm2
+; KNL-NEXT: vpcmpeqb %ymm4, %ymm1, %ymm1
+; KNL-NEXT: vpcmpeqb %ymm4, %ymm2, %ymm2
; KNL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; KNL-NEXT: vextracti64x4 $1, %zmm3, %ymm2
-; KNL-NEXT: vpcmpeqb %ymm5, %ymm2, %ymm2
-; KNL-NEXT: vpcmpeqb %ymm5, %ymm3, %ymm3
+; KNL-NEXT: vpcmpeqb %ymm4, %ymm2, %ymm2
+; KNL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm3
; KNL-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2
; KNL-NEXT: vpternlogq {{.*#+}} zmm2 = zmm0 & (zmm2 | zmm1)
; KNL-NEXT: vextracti64x4 $1, %zmm2, %ymm0
@@ -4331,23 +4331,23 @@ define void @ktest_7(<64 x i8> %w, <64 x i8> %x, <64 x i8> %y, <64 x i8> %z) {
;
; AVX512DQ-LABEL: ktest_7:
; AVX512DQ: ## %bb.0:
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm4
-; AVX512DQ-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX512DQ-NEXT: vpcmpeqb %ymm5, %ymm4, %ymm4
-; AVX512DQ-NEXT: vpcmpeqb %ymm5, %ymm0, %ymm0
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm0, %zmm0
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm4
-; AVX512DQ-NEXT: vpcmpeqb %ymm5, %ymm4, %ymm4
-; AVX512DQ-NEXT: vpcmpeqb %ymm5, %ymm1, %ymm1
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm4, %zmm1, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm5
+; AVX512DQ-NEXT: vpcmpeqb %ymm4, %ymm5, %ymm5
+; AVX512DQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm0, %zmm0
+; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm5
+; AVX512DQ-NEXT: vpcmpeqb %ymm4, %ymm5, %ymm5
+; AVX512DQ-NEXT: vpcmpeqb %ymm4, %ymm1, %ymm1
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm5, %zmm1, %zmm1
; AVX512DQ-NEXT: vporq %zmm1, %zmm0, %zmm0
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm2, %ymm1
-; AVX512DQ-NEXT: vpcmpeqb %ymm5, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpcmpeqb %ymm5, %ymm2, %ymm2
+; AVX512DQ-NEXT: vpcmpeqb %ymm4, %ymm1, %ymm1
+; AVX512DQ-NEXT: vpcmpeqb %ymm4, %ymm2, %ymm2
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm3, %ymm2
-; AVX512DQ-NEXT: vpcmpeqb %ymm5, %ymm2, %ymm2
-; AVX512DQ-NEXT: vpcmpeqb %ymm5, %ymm3, %ymm3
+; AVX512DQ-NEXT: vpcmpeqb %ymm4, %ymm2, %ymm2
+; AVX512DQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm3
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm3, %zmm2
; AVX512DQ-NEXT: vpternlogq {{.*#+}} zmm2 = zmm0 & (zmm2 | zmm1)
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm2, %ymm0
diff --git a/llvm/test/CodeGen/X86/avx512-mask-set-opt.ll b/llvm/test/CodeGen/X86/avx512-mask-set-opt.ll
index ca5f3192d7b97..24a03d0381ebb 100644
--- a/llvm/test/CodeGen/X86/avx512-mask-set-opt.ll
+++ b/llvm/test/CodeGen/X86/avx512-mask-set-opt.ll
@@ -79,8 +79,8 @@ define <16 x float> @mask_v16i1_lower8(ptr %ptr) {
define <16 x float> @gather_all(ptr %base, <16 x i32> %ind, i16 %mask) {
; AVX512-LABEL: gather_all:
; AVX512: # %bb.0:
-; AVX512-NEXT: kxnorw %k0, %k0, %k1
; AVX512-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: kxnorw %k0, %k0, %k1
; AVX512-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}
; AVX512-NEXT: vmovaps %zmm1, %zmm0
; AVX512-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/avx512-select.ll b/llvm/test/CodeGen/X86/avx512-select.ll
index 721ffbe1ceb79..9ec8b5c0c2d85 100644
--- a/llvm/test/CodeGen/X86/avx512-select.ll
+++ b/llvm/test/CodeGen/X86/avx512-select.ll
@@ -8,22 +8,24 @@ define <16 x i32> @select00(i32 %a, <16 x i32> %b) nounwind {
; X86-LABEL: select00:
; X86: # %bb.0:
; X86-NEXT: cmpl $255, {{[0-9]+}}(%esp)
+; X86-NEXT: je .LBB0_1
+; X86-NEXT: # %bb.2:
+; X86-NEXT: vpxord %zmm0, %zmm0, %zmm0
+; X86-NEXT: retl
+; X86-NEXT: .LBB0_1:
; X86-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-NEXT: je .LBB0_2
-; X86-NEXT: # %bb.1:
-; X86-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86-NEXT: .LBB0_2:
; X86-NEXT: vpxord %zmm1, %zmm0, %zmm0
; X86-NEXT: retl
;
; X64-LABEL: select00:
; X64: # %bb.0:
; X64-NEXT: cmpl $255, %edi
+; X64-NEXT: je .LBB0_1
+; X64-NEXT: # %bb.2:
+; X64-NEXT: vpxord %zmm0, %zmm0, %zmm0
+; X64-NEXT: retq
+; X64-NEXT: .LBB0_1:
; X64-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-NEXT: je .LBB0_2
-; X64-NEXT: # %bb.1:
-; X64-NEXT: vmovdqa64 %zmm0, %zmm1
-; X64-NEXT: .LBB0_2:
; X64-NEXT: vpxord %zmm1, %zmm0, %zmm0
; X64-NEXT: retq
%cmpres = icmp eq i32 %a, 255
@@ -36,22 +38,24 @@ define <8 x i64> @select01(i32 %a, <8 x i64> %b) nounwind {
; X86-LABEL: select01:
; X86: # %bb.0:
; X86-NEXT: cmpl $255, {{[0-9]+}}(%esp)
+; X86-NEXT: je .LBB1_1
+; X86-NEXT: # %bb.2:
+; X86-NEXT: vpxorq %zmm0, %zmm0, %zmm0
+; X86-NEXT: retl
+; X86-NEXT: .LBB1_1:
; X86-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-NEXT: je .LBB1_2
-; X86-NEXT: # %bb.1:
-; X86-NEXT: vmovdqa64 %zmm0, %zmm1
-; X86-NEXT: .LBB1_2:
; X86-NEXT: vpxorq %zmm1, %zmm0, %zmm0
; X86-NEXT: retl
;
; X64-LABEL: select01:
; X64: # %bb.0:
; X64-NEXT: cmpl $255, %edi
+; X64-NEXT: je .LBB1_1
+; X64-NEXT: # %bb.2:
+; X64-NEXT: vpxorq %zmm0, %zmm0, %zmm0
+; X64-NEXT: retq
+; X64-NEXT: .LBB1_1:
; X64-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-NEXT: je .LBB1_2
-; X64-NEXT: # %bb.1:
-; X64-NEXT: vmovdqa64 %zmm0, %zmm1
-; X64-NEXT: .LBB1_2:
; X64-NEXT: vpxorq %zmm1, %zmm0, %zmm0
; X64-NEXT: retq
%cmpres = icmp eq i32 %a, 255
diff --git a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
index 78b0199f2ed38..c95f177d2d780 100644
--- a/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
+++ b/llvm/test/CodeGen/X86/avx512-vec-cmp.ll
@@ -1495,8 +1495,8 @@ define void @half_vec_compare(ptr %x, ptr %y) {
; SKX: ## %bb.0: ## %entry
; SKX-NEXT: vmovss {{.*#+}} xmm0 = mem[0],zero,zero,zero
; SKX-NEXT: ## EVEX TO VEX Compression encoding: [0xc5,0xfa,0x10,0x07]
-; SKX-NEXT: vcvtph2ps %xmm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x13,0xc0]
; SKX-NEXT: vxorps %xmm1, %xmm1, %xmm1 ## EVEX TO VEX Compression encoding: [0xc5,0xf0,0x57,0xc9]
+; SKX-NEXT: vcvtph2ps %xmm0, %ymm0 ## EVEX TO VEX Compression encoding: [0xc4,0xe2,0x7d,0x13,0xc0]
; SKX-NEXT: vcmpneqps %ymm1, %ymm0, %k1 ## encoding: [0x62,0xf1,0x7c,0x28,0xc2,0xc9,0x04]
; SKX-NEXT: vmovdqu8 {{.*#+}} xmm0 {%k1} {z} = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; SKX-NEXT: ## encoding: [0x62,0xf1,0x7f,0x89,0x6f,0x05,A,A,A,A]
diff --git a/llvm/test/CodeGen/X86/avx512-vselect.ll b/llvm/test/CodeGen/X86/avx512-vselect.ll
index aacd5d1ab0298..c990a80d03b3a 100644
--- a/llvm/test/CodeGen/X86/avx512-vselect.ll
+++ b/llvm/test/CodeGen/X86/avx512-vselect.ll
@@ -146,12 +146,12 @@ define <32 x i32> @test7(<32 x i16> %x, <32 x i32> %a, <32 x i32> %b) {
;
; CHECK-KNL-LABEL: test7:
; CHECK-KNL: # %bb.0:
-; CHECK-KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm5
-; CHECK-KNL-NEXT: vpxor %xmm6, %xmm6, %xmm6
-; CHECK-KNL-NEXT: vpcmpeqw %ymm6, %ymm5, %ymm5
-; CHECK-KNL-NEXT: vpmovsxwd %ymm5, %zmm5
-; CHECK-KNL-NEXT: vptestmd %zmm5, %zmm5, %k1
-; CHECK-KNL-NEXT: vpcmpeqw %ymm6, %ymm0, %ymm0
+; CHECK-KNL-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; CHECK-KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm6
+; CHECK-KNL-NEXT: vpcmpeqw %ymm5, %ymm6, %ymm6
+; CHECK-KNL-NEXT: vpmovsxwd %ymm6, %zmm6
+; CHECK-KNL-NEXT: vptestmd %zmm6, %zmm6, %k1
+; CHECK-KNL-NEXT: vpcmpeqw %ymm5, %ymm0, %ymm0
; CHECK-KNL-NEXT: vpmovsxwd %ymm0, %zmm0
; CHECK-KNL-NEXT: vptestmd %zmm0, %zmm0, %k2
; CHECK-KNL-NEXT: vpblendmd %zmm1, %zmm3, %zmm0 {%k2}
@@ -173,17 +173,17 @@ define <64 x i16> @test8(<64 x i8> %x, <64 x i16> %a, <64 x i16> %b) {
;
; CHECK-KNL-LABEL: test8:
; CHECK-KNL: # %bb.0:
-; CHECK-KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm5
-; CHECK-KNL-NEXT: vpxor %xmm6, %xmm6, %xmm6
-; CHECK-KNL-NEXT: vpcmpeqb %ymm6, %ymm5, %ymm5
-; CHECK-KNL-NEXT: vpcmpeqb %ymm6, %ymm0, %ymm0
-; CHECK-KNL-NEXT: vpmovsxbw %xmm0, %ymm6
+; CHECK-KNL-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; CHECK-KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm6
+; CHECK-KNL-NEXT: vpcmpeqb %ymm5, %ymm6, %ymm6
+; CHECK-KNL-NEXT: vpcmpeqb %ymm5, %ymm0, %ymm0
+; CHECK-KNL-NEXT: vpmovsxbw %xmm0, %ymm5
; CHECK-KNL-NEXT: vextracti128 $1, %ymm0, %xmm0
; CHECK-KNL-NEXT: vpmovsxbw %xmm0, %ymm0
-; CHECK-KNL-NEXT: vinserti64x4 $1, %ymm0, %zmm6, %zmm0
+; CHECK-KNL-NEXT: vinserti64x4 $1, %ymm0, %zmm5, %zmm0
; CHECK-KNL-NEXT: vpternlogq {{.*#+}} zmm0 = zmm3 ^ (zmm0 & (zmm1 ^ zmm3))
-; CHECK-KNL-NEXT: vpmovsxbw %xmm5, %ymm1
-; CHECK-KNL-NEXT: vextracti128 $1, %ymm5, %xmm3
+; CHECK-KNL-NEXT: vpmovsxbw %xmm6, %ymm1
+; CHECK-KNL-NEXT: vextracti128 $1, %ymm6, %xmm3
; CHECK-KNL-NEXT: vpmovsxbw %xmm3, %ymm3
; CHECK-KNL-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
; CHECK-KNL-NEXT: vpternlogq {{.*#+}} zmm1 = zmm4 ^ (zmm1 & (zmm2 ^ zmm4))
diff --git a/llvm/test/CodeGen/X86/avx512f-256-set0.mir b/llvm/test/CodeGen/X86/avx512f-256-set0.mir
index a77c724b5063a..93c6e68badc31 100644
--- a/llvm/test/CodeGen/X86/avx512f-256-set0.mir
+++ b/llvm/test/CodeGen/X86/avx512f-256-set0.mir
@@ -62,9 +62,11 @@ body: |
bb.0.bb0:
; CHECK-LABEL: name: main
; CHECK: $zmm16 = VPXORDZrr undef $zmm16, undef $zmm16
- ; CHECK: VMOVAPSZmr $rip, 1, $noreg, @tst_, $noreg, killed renamable $zmm16 :: (store (s256) into %ir.lsr.iv1, align 64)
- ; CHECK: RET 0
- renamable $ymm16 = AVX512_256_SET0
+ ; CHECK-NEXT: renamable $ymm16 = KILL $xmm16
+ ; CHECK-NEXT: VMOVAPSZmr $rip, 1, $noreg, @tst_, $noreg, killed renamable $zmm16 :: (store (s256) into %ir.lsr.iv1, align 64)
+ ; CHECK-NEXT: RET 0
+ renamable $xmm16 = AVX512_128_SET0
+ renamable $ymm16 = SUBREG_TO_REG $xmm16, %subreg.sub_xmm
VMOVAPSZmr $rip, 1, $noreg, @tst_, $noreg, killed renamable $zmm16 :: (store (s256) into %ir.lsr.iv1, align 64)
RET 0
diff --git a/llvm/test/CodeGen/X86/bit-manip-i256.ll b/llvm/test/CodeGen/X86/bit-manip-i256.ll
index 994443117c165..042b7d428ba5c 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i256.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i256.ll
@@ -81,10 +81,10 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq %rcx, %rax
; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
+; AVX2-NEXT: vmovss {{.*#+}} xmm0 = [1,0,0,0]
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovss {{.*#+}} xmm1 = [1,0,0,0]
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %r10d, %ecx
; AVX2-NEXT: shrb $3, %cl
; AVX2-NEXT: andb $24, %cl
@@ -104,11 +104,11 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX2-NEXT: adcq $-1, %r14
; AVX2-NEXT: adcq $-1, %rbx
; AVX2-NEXT: adcq $-1, %r11
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $6, %al
; AVX2-NEXT: movzbl %al, %edx
@@ -163,12 +163,12 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512F-NEXT: adcq $-1, %r11
; AVX512F-NEXT: adcq $-1, %r10
; AVX512F-NEXT: movq %r9, %rcx
-; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movl %ecx, %eax
; AVX512F-NEXT: shrb $6, %al
; AVX512F-NEXT: movzbl %al, %edx
@@ -203,10 +203,10 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VL-NEXT: movq %rcx, %r10
; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: movzbl {{[0-9]+}}(%rsp), %ecx
+; AVX512VL-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovaps {{.*#+}} xmm1 = [1,0,0,0]
-; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %ecx, %edi
; AVX512VL-NEXT: shrb $3, %dil
; AVX512VL-NEXT: andb $24, %dil
@@ -224,11 +224,11 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VL-NEXT: adcq $-1, %r14
; AVX512VL-NEXT: adcq $-1, %r11
; AVX512VL-NEXT: adcq $-1, %rdi
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %r9d, %ecx
; AVX512VL-NEXT: shrb $6, %cl
; AVX512VL-NEXT: movzbl %cl, %edx
@@ -264,10 +264,10 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VBMI-NEXT: movq %rcx, %r10
; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: movzbl {{[0-9]+}}(%rsp), %ecx
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovaps {{.*#+}} xmm1 = [1,0,0,0]
-; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %ecx, %edi
; AVX512VBMI-NEXT: shrb $3, %dil
; AVX512VBMI-NEXT: andb $24, %dil
@@ -285,11 +285,11 @@ define i256 @bext_i256(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VBMI-NEXT: adcq $-1, %r14
; AVX512VBMI-NEXT: adcq $-1, %r11
; AVX512VBMI-NEXT: adcq $-1, %rdi
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %r9d, %ecx
; AVX512VBMI-NEXT: shrb $6, %cl
; AVX512VBMI-NEXT: movzbl %cl, %edx
@@ -395,10 +395,10 @@ define i256 @bext_i256_vector(<4 x i64> %v0, i256 %idx, i256 %len) nounwind {
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq %r9, %rcx
+; AVX2-NEXT: vmovss {{.*#+}} xmm1 = [1,0,0,0]
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovss {{.*#+}} xmm2 = [1,0,0,0]
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $3, %al
; AVX2-NEXT: andb $24, %al
@@ -416,8 +416,8 @@ define i256 @bext_i256_vector(<4 x i64> %v0, i256 %idx, i256 %len) nounwind {
; AVX2-NEXT: adcq $-1, %r10
; AVX2-NEXT: adcq $-1, %r8
; AVX2-NEXT: adcq $-1, %rdx
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %esi, %eax
; AVX2-NEXT: shrb $6, %al
; AVX2-NEXT: movzbl %al, %r11d
@@ -470,10 +470,10 @@ define i256 @bext_i256_vector(<4 x i64> %v0, i256 %idx, i256 %len) nounwind {
; AVX512F-NEXT: addq $-1, %r9
; AVX512F-NEXT: adcq $-1, %r10
; AVX512F-NEXT: adcq $-1, %r8
-; AVX512F-NEXT: movq %rsi, %rcx
; AVX512F-NEXT: adcq $-1, %rdx
-; AVX512F-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: movq %rsi, %rcx
+; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movl %ecx, %eax
; AVX512F-NEXT: shrb $6, %al
@@ -507,10 +507,10 @@ define i256 @bext_i256_vector(<4 x i64> %v0, i256 %idx, i256 %len) nounwind {
; AVX512VL-NEXT: pushq %r14
; AVX512VL-NEXT: pushq %rbx
; AVX512VL-NEXT: movq %r9, %rcx
+; AVX512VL-NEXT: vmovaps {{.*#+}} xmm1 = [1,0,0,0]
+; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovaps {{.*#+}} xmm2 = [1,0,0,0]
-; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %ecx, %eax
; AVX512VL-NEXT: shrb $3, %al
; AVX512VL-NEXT: andb $24, %al
@@ -528,9 +528,9 @@ define i256 @bext_i256_vector(<4 x i64> %v0, i256 %idx, i256 %len) nounwind {
; AVX512VL-NEXT: addq $-1, %rdi
; AVX512VL-NEXT: adcq $-1, %r9
; AVX512VL-NEXT: adcq $-1, %r8
-; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: adcq $-1, %rdx
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: adcq $-1, %rdx
+; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %esi, %ecx
; AVX512VL-NEXT: shrb $6, %cl
; AVX512VL-NEXT: movzbl %cl, %r10d
@@ -564,10 +564,10 @@ define i256 @bext_i256_vector(<4 x i64> %v0, i256 %idx, i256 %len) nounwind {
; AVX512VBMI-NEXT: pushq %r14
; AVX512VBMI-NEXT: pushq %rbx
; AVX512VBMI-NEXT: movq %r9, %rcx
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} xmm1 = [1,0,0,0]
+; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovaps {{.*#+}} xmm2 = [1,0,0,0]
-; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %ecx, %eax
; AVX512VBMI-NEXT: shrb $3, %al
; AVX512VBMI-NEXT: andb $24, %al
@@ -585,9 +585,9 @@ define i256 @bext_i256_vector(<4 x i64> %v0, i256 %idx, i256 %len) nounwind {
; AVX512VBMI-NEXT: addq $-1, %rdi
; AVX512VBMI-NEXT: adcq $-1, %r9
; AVX512VBMI-NEXT: adcq $-1, %r8
-; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: adcq $-1, %rdx
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: adcq $-1, %rdx
+; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %esi, %ecx
; AVX512VBMI-NEXT: shrb $6, %cl
; AVX512VBMI-NEXT: movzbl %cl, %r10d
@@ -697,10 +697,10 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq %rcx, %rax
; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %r10d
+; AVX2-NEXT: vmovss {{.*#+}} xmm0 = [1,0,0,0]
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovss {{.*#+}} xmm1 = [1,0,0,0]
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %r10d, %ecx
; AVX2-NEXT: shrb $3, %cl
; AVX2-NEXT: andb $24, %cl
@@ -720,11 +720,11 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX2-NEXT: adcq $-1, %r14
; AVX2-NEXT: adcq $-1, %rbx
; AVX2-NEXT: adcq $-1, %r11
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $6, %al
; AVX2-NEXT: movzbl %al, %edx
@@ -779,12 +779,12 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512F-NEXT: adcq $-1, %r11
; AVX512F-NEXT: adcq $-1, %r10
; AVX512F-NEXT: movq %r9, %rcx
-; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movl %ecx, %eax
; AVX512F-NEXT: shrb $6, %al
; AVX512F-NEXT: movzbl %al, %edx
@@ -819,10 +819,10 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VL-NEXT: movq %rcx, %r10
; AVX512VL-NEXT: movq %rdi, %rax
; AVX512VL-NEXT: movzbl {{[0-9]+}}(%rsp), %ecx
+; AVX512VL-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovaps {{.*#+}} xmm1 = [1,0,0,0]
-; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %ecx, %edi
; AVX512VL-NEXT: shrb $3, %dil
; AVX512VL-NEXT: andb $24, %dil
@@ -840,11 +840,11 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VL-NEXT: adcq $-1, %r14
; AVX512VL-NEXT: adcq $-1, %r11
; AVX512VL-NEXT: adcq $-1, %rdi
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %r9d, %ecx
; AVX512VL-NEXT: shrb $6, %cl
; AVX512VL-NEXT: movzbl %cl, %edx
@@ -880,10 +880,10 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VBMI-NEXT: movq %rcx, %r10
; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: movzbl {{[0-9]+}}(%rsp), %ecx
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovaps {{.*#+}} xmm1 = [1,0,0,0]
-; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %ecx, %edi
; AVX512VBMI-NEXT: shrb $3, %dil
; AVX512VBMI-NEXT: andb $24, %dil
@@ -901,11 +901,11 @@ define i256 @bext_i256_load(i256 %a0, i256 %idx, i256 %len) nounwind {
; AVX512VBMI-NEXT: adcq $-1, %r14
; AVX512VBMI-NEXT: adcq $-1, %r11
; AVX512VBMI-NEXT: adcq $-1, %rdi
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %r9d, %ecx
; AVX512VBMI-NEXT: shrb $6, %cl
; AVX512VBMI-NEXT: movzbl %cl, %edx
@@ -1795,10 +1795,10 @@ define i256 @bzhi_i256(i256 %a0, i256 %idx) nounwind {
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: movq %rcx, %rax
-; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovss {{.*#+}} xmm0 = [1,0,0,0]
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %r9d, %ecx
; AVX2-NEXT: shrb $3, %cl
; AVX2-NEXT: andb $24, %cl
@@ -1874,10 +1874,10 @@ define i256 @bzhi_i256(i256 %a0, i256 %idx) nounwind {
; AVX512VL-NEXT: pushq %r14
; AVX512VL-NEXT: pushq %rbx
; AVX512VL-NEXT: movq %rcx, %rax
-; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %r9d, %ecx
; AVX512VL-NEXT: shrb $3, %cl
; AVX512VL-NEXT: andb $24, %cl
@@ -1915,10 +1915,10 @@ define i256 @bzhi_i256(i256 %a0, i256 %idx) nounwind {
; AVX512VBMI-NEXT: pushq %r14
; AVX512VBMI-NEXT: pushq %rbx
; AVX512VBMI-NEXT: movq %rcx, %rax
-; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %r9d, %ecx
; AVX512VBMI-NEXT: shrb $3, %cl
; AVX512VBMI-NEXT: andb $24, %cl
@@ -2046,10 +2046,10 @@ define i256 @bzhi_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX2-LABEL: bzhi_i256_vector:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovss {{.*#+}} xmm1 = [1,0,0,0]
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, %rcx
; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: vmovq %xmm0, %rdx
@@ -2131,10 +2131,10 @@ define i256 @bzhi_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX512VL-NEXT: pushq %rbx
; AVX512VL-NEXT: movq %rsi, %rcx
; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps {{.*#+}} xmm1 = [1,0]
; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovq %xmm0, %rdx
; AVX512VL-NEXT: vpextrq $1, %xmm0, %rsi
; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm0
@@ -2174,10 +2174,10 @@ define i256 @bzhi_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX512VBMI-NEXT: pushq %rbx
; AVX512VBMI-NEXT: movq %rsi, %rcx
; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps {{.*#+}} xmm1 = [1,0]
; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovq %xmm0, %rdx
; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rsi
; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm0
@@ -2260,10 +2260,10 @@ define i256 @bzhi_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX2-LABEL: bzhi_i256_load:
; AVX2: # %bb.0:
; AVX2-NEXT: movq %rdx, %rcx
-; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovss {{.*#+}} xmm0 = [1,0,0,0]
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $3, %al
; AVX2-NEXT: andb $24, %al
@@ -2329,10 +2329,10 @@ define i256 @bzhi_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX512VL-LABEL: bzhi_i256_load:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: movq %rdx, %rcx
-; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %ecx, %eax
; AVX512VL-NEXT: shrb $3, %al
; AVX512VL-NEXT: andb $24, %al
@@ -2365,10 +2365,10 @@ define i256 @bzhi_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX512VBMI-LABEL: bzhi_i256_load:
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: movq %rdx, %rcx
-; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %ecx, %eax
; AVX512VBMI-NEXT: shrb $3, %al
; AVX512VBMI-NEXT: andb $24, %al
@@ -2486,10 +2486,10 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; AVX2-NEXT: subq $-128, %rcx
; AVX2-NEXT: orq %r8, %rax
; AVX2-NEXT: cmovneq %r11, %rcx
-; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $6, %al
; AVX2-NEXT: movzbl %al, %eax
@@ -2586,11 +2586,11 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; AVX512VL-NEXT: cmovneq %r10, %rcx
; AVX512VL-NEXT: subq $-128, %rcx
; AVX512VL-NEXT: orq %r8, %rax
+; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: cmovneq %r11, %rcx
; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %ecx, %eax
; AVX512VL-NEXT: shrb $6, %al
; AVX512VL-NEXT: movzbl %al, %eax
@@ -2638,11 +2638,11 @@ define i256 @isolate_msb_i256(i256 %a0, i256 %idx) nounwind {
; AVX512VBMI-NEXT: cmovneq %r10, %rcx
; AVX512VBMI-NEXT: subq $-128, %rcx
; AVX512VBMI-NEXT: orq %r8, %rax
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: cmovneq %r11, %rcx
; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %ecx, %eax
; AVX512VBMI-NEXT: shrb $6, %al
; AVX512VBMI-NEXT: movzbl %al, %eax
@@ -2823,10 +2823,10 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX2-NEXT: subq $-128, %rcx
; AVX2-NEXT: orq %r8, %rsi
; AVX2-NEXT: cmovneq %r9, %rcx
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $6, %al
; AVX2-NEXT: movzbl %al, %edx
@@ -2901,10 +2901,10 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm1 {%k1}
-; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovd %xmm1, %ecx
; AVX512VL-NEXT: movl %ecx, %eax
; AVX512VL-NEXT: shrb $6, %al
@@ -2940,10 +2940,10 @@ define i256 @isolate_msb_i256_vector(<4 x i64> %v0, i256 %idx) nounwind {
; AVX512VBMI-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VBMI-NEXT: vpcompressq %ymm1, %ymm1 {%k1}
-; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovd %xmm1, %ecx
; AVX512VBMI-NEXT: movl %ecx, %eax
; AVX512VBMI-NEXT: shrb $6, %al
@@ -3120,10 +3120,10 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX2-NEXT: orq %rdx, %rax
; AVX2-NEXT: cmovneq %r8, %rcx
; AVX2-NEXT: vmovdqu (%rsi), %ymm0
-; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $6, %al
; AVX2-NEXT: movzbl %al, %edx
@@ -3199,10 +3199,10 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX512VL-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VL-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VL-NEXT: vpcompressq %ymm1, %ymm1 {%k1} {z}
-; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovd %xmm1, %ecx
; AVX512VL-NEXT: movl %ecx, %eax
; AVX512VL-NEXT: shrb $6, %al
@@ -3239,10 +3239,10 @@ define i256 @isolate_msb_i256_load(ptr %p0, i256 %idx) nounwind {
; AVX512VBMI-NEXT: vplzcntq %ymm1, %ymm1
; AVX512VBMI-NEXT: vpaddq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1 # [0,64,128,192]
; AVX512VBMI-NEXT: vpcompressq %ymm1, %ymm1 {%k1} {z}
-; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512VBMI-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovd %xmm1, %ecx
; AVX512VBMI-NEXT: movl %ecx, %eax
; AVX512VBMI-NEXT: shrb $6, %al
diff --git a/llvm/test/CodeGen/X86/bit-manip-i512.ll b/llvm/test/CodeGen/X86/bit-manip-i512.ll
index 461b5eed3250b..9e4d00650e613 100644
--- a/llvm/test/CodeGen/X86/bit-manip-i512.ll
+++ b/llvm/test/CodeGen/X86/bit-manip-i512.ll
@@ -136,92 +136,91 @@ define i512 @bext_i512(i512 %a0, i512 %idx, i512 %len) nounwind {
; AVX2-NEXT: subq $168, %rsp
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, {{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm0, {{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm0, {{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vmovss {{.*#+}} xmm1 = [1,0,0,0]
; AVX2-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
-; AVX2-NEXT: movl %eax, %ecx
+; AVX2-NEXT: vmovups %ymm0, {{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm0, {{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl {{[0-9]+}}(%rsp), %r10d
+; AVX2-NEXT: movl %r10d, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: shrl $3, %eax
-; AVX2-NEXT: andl $56, %eax
-; AVX2-NEXT: negl %eax
-; AVX2-NEXT: movslq %eax, %r10
-; AVX2-NEXT: movq 144(%rsp,%r10), %r11
-; AVX2-NEXT: movq 152(%rsp,%r10), %rax
-; AVX2-NEXT: shldq %cl, %r11, %rax
-; AVX2-NEXT: movq 136(%rsp,%r10), %rbx
-; AVX2-NEXT: shldq %cl, %rbx, %r11
-; AVX2-NEXT: movq 128(%rsp,%r10), %r14
+; AVX2-NEXT: shrl $3, %r10d
+; AVX2-NEXT: andl $56, %r10d
+; AVX2-NEXT: negl %r10d
+; AVX2-NEXT: movslq %r10d, %r10
+; AVX2-NEXT: movq 144(%rsp,%r10), %r14
+; AVX2-NEXT: movq 152(%rsp,%r10), %rbx
; AVX2-NEXT: shldq %cl, %r14, %rbx
+; AVX2-NEXT: movq 136(%rsp,%r10), %rax
+; AVX2-NEXT: shldq %cl, %rax, %r14
+; AVX2-NEXT: movq 128(%rsp,%r10), %r11
+; AVX2-NEXT: shldq %cl, %r11, %rax
; AVX2-NEXT: movq 120(%rsp,%r10), %r12
-; AVX2-NEXT: shldq %cl, %r12, %r14
+; AVX2-NEXT: shldq %cl, %r12, %r11
; AVX2-NEXT: movq 112(%rsp,%r10), %r13
; AVX2-NEXT: shldq %cl, %r13, %r12
-; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: movq 96(%rsp,%r10), %rdx
+; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm1
+; AVX2-NEXT: movq 96(%rsp,%r10), %r15
; AVX2-NEXT: movq 104(%rsp,%r10), %rbp
; AVX2-NEXT: shldq %cl, %rbp, %r13
-; AVX2-NEXT: shldq %cl, %rdx, %rbp
-; AVX2-NEXT: shlxq %rcx, %rdx, %rdx
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
-; AVX2-NEXT: vmovups %ymm0, (%rsp)
-; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl {{[0-9]+}}(%rsp), %r15d
-; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm1
-; AVX2-NEXT: addq $-1, %rdx
-; AVX2-NEXT: movq %rdx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: shldq %cl, %r15, %rbp
+; AVX2-NEXT: shlxq %rcx, %r15, %rcx
+; AVX2-NEXT: addq $-1, %rcx
+; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: adcq $-1, %rbp
; AVX2-NEXT: adcq $-1, %r13
; AVX2-NEXT: adcq $-1, %r12
-; AVX2-NEXT: adcq $-1, %r14
-; AVX2-NEXT: adcq $-1, %rbx
; AVX2-NEXT: adcq $-1, %r11
; AVX2-NEXT: adcq $-1, %rax
; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: adcq $-1, %r14
+; AVX2-NEXT: adcq $-1, %rbx
+; AVX2-NEXT: movq %rbx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %xmm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
; AVX2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %r15d, %ecx
+; AVX2-NEXT: vmovups %ymm0, (%rsp)
+; AVX2-NEXT: movl {{[0-9]+}}(%rsp), %r8d
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %r8d, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: shrl $3, %r15d
-; AVX2-NEXT: andl $56, %r15d
-; AVX2-NEXT: movq -80(%rsp,%r15), %rsi
-; AVX2-NEXT: movq -88(%rsp,%r15), %rdx
-; AVX2-NEXT: movq %rdx, %r8
-; AVX2-NEXT: shrdq %cl, %rsi, %r8
-; AVX2-NEXT: movq -72(%rsp,%r15), %r9
-; AVX2-NEXT: shrdq %cl, %r9, %rsi
-; AVX2-NEXT: movq -64(%rsp,%r15), %r10
+; AVX2-NEXT: shrl $3, %r8d
+; AVX2-NEXT: andl $56, %r8d
+; AVX2-NEXT: movq -80(%rsp,%r8), %rdx
+; AVX2-NEXT: movq -88(%rsp,%r8), %r15
+; AVX2-NEXT: movq %r15, %rsi
+; AVX2-NEXT: shrdq %cl, %rdx, %rsi
+; AVX2-NEXT: movq -72(%rsp,%r8), %r9
+; AVX2-NEXT: shrdq %cl, %r9, %rdx
+; AVX2-NEXT: movq -64(%rsp,%r8), %r10
; AVX2-NEXT: shrdq %cl, %r10, %r9
; AVX2-NEXT: movq %rdi, %rax
-; AVX2-NEXT: movq -56(%rsp,%r15), %rdi
+; AVX2-NEXT: movq -56(%rsp,%r8), %rdi
; AVX2-NEXT: shrdq %cl, %rdi, %r10
-; AVX2-NEXT: andq %rbp, %r8
-; AVX2-NEXT: andq %r13, %rsi
+; AVX2-NEXT: andq %rbp, %rsi
+; AVX2-NEXT: andq %r13, %rdx
; AVX2-NEXT: andq %r12, %r9
-; AVX2-NEXT: movq -48(%rsp,%r15), %r12
+; AVX2-NEXT: movq -48(%rsp,%r8), %r12
; AVX2-NEXT: shrdq %cl, %r12, %rdi
-; AVX2-NEXT: andq %r14, %r10
-; AVX2-NEXT: andq %rbx, %rdi
-; AVX2-NEXT: movq -96(%rsp,%r15), %rbx
-; AVX2-NEXT: movq -40(%rsp,%r15), %r14
-; AVX2-NEXT: shrdq %cl, %r14, %r12
-; AVX2-NEXT: shrdq %cl, %rdx, %rbx
-; AVX2-NEXT: andq %r11, %r12
+; AVX2-NEXT: andq %r11, %r10
+; AVX2-NEXT: andq {{[-0-9]+}}(%r{{[sb]}}p), %rdi # 8-byte Folded Reload
+; AVX2-NEXT: movq -96(%rsp,%r8), %rbx
+; AVX2-NEXT: movq -40(%rsp,%r8), %r8
+; AVX2-NEXT: shrdq %cl, %r8, %r12
+; AVX2-NEXT: shrdq %cl, %r15, %rbx
+; AVX2-NEXT: andq %r14, %r12
; AVX2-NEXT: andq {{[-0-9]+}}(%r{{[sb]}}p), %rbx # 8-byte Folded Reload
-; AVX2-NEXT: shrxq %rcx, %r14, %rcx
+; AVX2-NEXT: shrxq %rcx, %r8, %rcx
; AVX2-NEXT: andq {{[-0-9]+}}(%r{{[sb]}}p), %rcx # 8-byte Folded Reload
; AVX2-NEXT: movq %rbx, (%rax)
-; AVX2-NEXT: movq %r8, 8(%rax)
-; AVX2-NEXT: movq %rsi, 16(%rax)
+; AVX2-NEXT: movq %rsi, 8(%rax)
+; AVX2-NEXT: movq %rdx, 16(%rax)
; AVX2-NEXT: movq %r9, 24(%rax)
; AVX2-NEXT: movq %r10, 32(%rax)
; AVX2-NEXT: movq %rdi, 40(%rax)
@@ -243,38 +242,37 @@ define i512 @bext_i512(i512 %a0, i512 %idx, i512 %len) nounwind {
; AVX512F-NEXT: pushq %rbx
; AVX512F-NEXT: pushq %rax
; AVX512F-NEXT: movq %rdi, %rax
+; AVX512F-NEXT: movl {{[0-9]+}}(%rsp), %edi
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
-; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; AVX512F-NEXT: movl $1, %r10d
-; AVX512F-NEXT: shlxq %rdi, %r10, %r11
-; AVX512F-NEXT: shrl $6, %edi
-; AVX512F-NEXT: shlxq %rdi, %r10, %rdi
-; AVX512F-NEXT: kmovw %edi, %k1
-; AVX512F-NEXT: vpbroadcastq %r11, %zmm2 {%k1} {z}
-; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: movl $1, %r11d
+; AVX512F-NEXT: shlxq %r10, %r11, %r14
+; AVX512F-NEXT: shrl $6, %r10d
+; AVX512F-NEXT: shlxq %r10, %r11, %r10
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rbx
+; AVX512F-NEXT: kmovw %r10d, %k1
+; AVX512F-NEXT: vpbroadcastq %r14, %zmm2 {%k1} {z}
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1
; AVX512F-NEXT: vpaddq %zmm1, %zmm2, %zmm1
; AVX512F-NEXT: vpcmpltuq %zmm2, %zmm1, %k0
-; AVX512F-NEXT: kmovw %k0, %edi
-; AVX512F-NEXT: vptestnmq %zmm2, %zmm2, %k0
; AVX512F-NEXT: kmovw %k0, %r10d
-; AVX512F-NEXT: movzbl %r10b, %r10d
-; AVX512F-NEXT: leal (%r10,%rdi,2), %edi
-; AVX512F-NEXT: xorl %r10d, %edi
-; AVX512F-NEXT: kmovw %edi, %k1
+; AVX512F-NEXT: vptestnmq %zmm2, %zmm2, %k0
+; AVX512F-NEXT: kmovw %k0, %r11d
+; AVX512F-NEXT: movzbl %r11b, %r11d
+; AVX512F-NEXT: leal (%r11,%r10,2), %r10d
+; AVX512F-NEXT: xorl %r11d, %r10d
+; AVX512F-NEXT: kmovw %r10d, %k1
; AVX512F-NEXT: vmovdqa64 %zmm2, %zmm1 {%k1}
; AVX512F-NEXT: vextracti32x4 $3, %zmm1, %xmm2
-; AVX512F-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512F-NEXT: vmovdqu64 %zmm3, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX512F-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vpextrq $1, %xmm2, %r11
+; AVX512F-NEXT: vmovq %xmm2, %r10
+; AVX512F-NEXT: vextracti32x4 $2, %zmm1, %xmm2
+; AVX512F-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vpextrq $1, %xmm2, %rbx
+; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512F-NEXT: vmovups %xmm0, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl {{[0-9]+}}(%rsp), %r14d
-; AVX512F-NEXT: vextracti32x4 $2, %zmm1, %xmm0
-; AVX512F-NEXT: vpextrq $1, %xmm0, %r10
-; AVX512F-NEXT: vmovq %xmm2, %rbx
; AVX512F-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vmovq %xmm0, %r11
+; AVX512F-NEXT: vmovq %xmm2, %r14
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: vpextrq $1, %xmm1, %r9
; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
@@ -282,14 +280,15 @@ define i512 @bext_i512(i512 %a0, i512 %idx, i512 %len) nounwind {
; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm0
; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX512F-NEXT: vmovdqu64 %zmm3, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movl %r14d, %edx
+; AVX512F-NEXT: movl %edi, %edx
; AVX512F-NEXT: andl $63, %edx
; AVX512F-NEXT: vmovq %rdx, %xmm1
; AVX512F-NEXT: vpbroadcastq %xmm1, %xmm1
-; AVX512F-NEXT: shrl $3, %r14d
-; AVX512F-NEXT: andl $56, %r14d
-; AVX512F-NEXT: vmovdqu64 -128(%rsp,%r14), %zmm2
+; AVX512F-NEXT: shrl $3, %edi
+; AVX512F-NEXT: andl $56, %edi
+; AVX512F-NEXT: vmovdqu64 -128(%rsp,%rdi), %zmm2
; AVX512F-NEXT: valignq {{.*#+}} zmm3 = zmm2[1,2,3,4,5,6,7],zmm3[0]
; AVX512F-NEXT: vpsrlq %xmm1, %zmm2, %zmm2
; AVX512F-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
@@ -298,14 +297,14 @@ define i512 @bext_i512(i512 %a0, i512 %idx, i512 %len) nounwind {
; AVX512F-NEXT: vporq %zmm2, %zmm1, %zmm1
; AVX512F-NEXT: vextracti32x4 $3, %zmm1, %xmm2
; AVX512F-NEXT: vpextrq $1, %xmm2, %rdx
-; AVX512F-NEXT: andq %rdi, %rdx
+; AVX512F-NEXT: andq %r11, %rdx
; AVX512F-NEXT: vmovq %xmm2, %rsi
; AVX512F-NEXT: vextracti32x4 $2, %zmm1, %xmm2
; AVX512F-NEXT: vpextrq $1, %xmm2, %rdi
-; AVX512F-NEXT: andq %rbx, %rsi
-; AVX512F-NEXT: andq %r10, %rdi
+; AVX512F-NEXT: andq %r10, %rsi
+; AVX512F-NEXT: andq %rbx, %rdi
; AVX512F-NEXT: vmovq %xmm2, %r10
-; AVX512F-NEXT: andq %r11, %r10
+; AVX512F-NEXT: andq %r14, %r10
; AVX512F-NEXT: vpextrq $1, %xmm1, %r11
; AVX512F-NEXT: andq %r9, %r11
; AVX512F-NEXT: vmovq %xmm1, %r9
@@ -331,20 +330,17 @@ define i512 @bext_i512(i512 %a0, i512 %idx, i512 %len) nounwind {
;
; AVX512VL-LABEL: bext_i512:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: pushq %r14
; AVX512VL-NEXT: pushq %rbx
-; AVX512VL-NEXT: pushq %rax
; AVX512VL-NEXT: movq %rdi, %rax
+; AVX512VL-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
; AVX512VL-NEXT: movl {{[0-9]+}}(%rsp), %edi
-; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512VL-NEXT: movl $1, %r11d
-; AVX512VL-NEXT: shlxq %r10, %r11, %r14
+; AVX512VL-NEXT: shlxq %r10, %r11, %rbx
; AVX512VL-NEXT: shrl $6, %r10d
; AVX512VL-NEXT: shlxq %r10, %r11, %r10
-; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX512VL-NEXT: kmovd %r10d, %k1
-; AVX512VL-NEXT: vpbroadcastq %r14, %zmm1 {%k1} {z}
+; AVX512VL-NEXT: vpbroadcastq %rbx, %zmm1 {%k1} {z}
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm2 = -1
; AVX512VL-NEXT: vpaddq %zmm2, %zmm1, %zmm2
; AVX512VL-NEXT: vpcmpltuq %zmm1, %zmm2, %k0
@@ -357,68 +353,66 @@ define i512 @bext_i512(i512 %a0, i512 %idx, i512 %len) nounwind {
; AVX512VL-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
; AVX512VL-NEXT: vextracti32x4 $3, %zmm2, %xmm1
; AVX512VL-NEXT: vpextrq $1, %xmm1, %r11
+; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %rbx
; AVX512VL-NEXT: vmovq %xmm1, %r10
; AVX512VL-NEXT: vextracti32x4 $2, %zmm2, %xmm1
-; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT: vmovdqu %ymm3, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vpextrq $1, %xmm1, %rbx
-; AVX512VL-NEXT: vmovdqu %ymm3, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovups %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovdqu %xmm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovq %xmm1, %r9
+; AVX512VL-NEXT: vpextrq $1, %xmm2, %r9
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vpextrq $1, %xmm2, %r8
; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: movl %edi, %ecx
-; AVX512VL-NEXT: andl $63, %ecx
-; AVX512VL-NEXT: vpbroadcastq %rcx, %xmm0
+; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovq %xmm1, %rcx
+; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: movl %edi, %edx
+; AVX512VL-NEXT: andl $63, %edx
+; AVX512VL-NEXT: vpbroadcastq %rdx, %xmm1
; AVX512VL-NEXT: shrl $3, %edi
; AVX512VL-NEXT: andl $56, %edi
-; AVX512VL-NEXT: vmovdqu64 -128(%rsp,%rdi), %zmm1
-; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT: valignq {{.*#+}} zmm3 = zmm1[1,2,3,4,5,6,7],zmm3[0]
-; AVX512VL-NEXT: vpsrlq %xmm0, %zmm1, %zmm1
-; AVX512VL-NEXT: vpaddq %zmm3, %zmm3, %zmm3
-; AVX512VL-NEXT: vpandnq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; AVX512VL-NEXT: vpsllq %xmm0, %zmm3, %zmm0
-; AVX512VL-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX512VL-NEXT: vmovdqu64 -128(%rsp,%rdi), %zmm3
+; AVX512VL-NEXT: valignq {{.*#+}} zmm0 = zmm3[1,2,3,4,5,6,7],zmm0[0]
+; AVX512VL-NEXT: vpsrlq %xmm1, %zmm3, %zmm3
+; AVX512VL-NEXT: vpaddq %zmm0, %zmm0, %zmm0
+; AVX512VL-NEXT: vpandnq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm1
+; AVX512VL-NEXT: vpsllq %xmm1, %zmm0, %zmm0
+; AVX512VL-NEXT: vextracti128 $1, %ymm2, %xmm1
+; AVX512VL-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512VL-NEXT: vporq %zmm3, %zmm0, %zmm0
+; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm3
; AVX512VL-NEXT: vpextrq $1, %xmm3, %rdx
-; AVX512VL-NEXT: vporq %zmm1, %zmm0, %zmm0
-; AVX512VL-NEXT: vextracti32x4 $3, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrq $1, %xmm1, %rcx
-; AVX512VL-NEXT: vmovq %xmm2, %rsi
-; AVX512VL-NEXT: andq %r11, %rcx
-; AVX512VL-NEXT: vmovq %xmm1, %rdi
-; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm1
-; AVX512VL-NEXT: vpextrq $1, %xmm1, %r11
-; AVX512VL-NEXT: andq %r10, %rdi
+; AVX512VL-NEXT: vmovq %xmm2, %rdi
+; AVX512VL-NEXT: andq %r11, %rdx
+; AVX512VL-NEXT: vmovq %xmm3, %r8
+; AVX512VL-NEXT: vextracti32x4 $2, %zmm0, %xmm2
+; AVX512VL-NEXT: vpextrq $1, %xmm2, %r11
+; AVX512VL-NEXT: andq %r10, %r8
; AVX512VL-NEXT: andq %rbx, %r11
-; AVX512VL-NEXT: vmovq %xmm1, %r10
-; AVX512VL-NEXT: andq %r9, %r10
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %r9
-; AVX512VL-NEXT: andq %r8, %r9
-; AVX512VL-NEXT: vmovq %xmm0, %r8
-; AVX512VL-NEXT: andq %rsi, %r8
+; AVX512VL-NEXT: vmovq %xmm2, %r10
+; AVX512VL-NEXT: andq %rcx, %r10
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %rcx
+; AVX512VL-NEXT: andq %r9, %rcx
+; AVX512VL-NEXT: vmovq %xmm0, %r9
+; AVX512VL-NEXT: andq %rdi, %r9
; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512VL-NEXT: vpextrq $1, %xmm0, %rsi
-; AVX512VL-NEXT: vmovq %xmm3, %rbx
-; AVX512VL-NEXT: andq %rdx, %rsi
-; AVX512VL-NEXT: vmovq %xmm0, %rdx
-; AVX512VL-NEXT: andq %rbx, %rdx
-; AVX512VL-NEXT: movq %rcx, 56(%rax)
-; AVX512VL-NEXT: movq %rdi, 48(%rax)
+; AVX512VL-NEXT: vpextrq $1, %xmm0, %rdi
+; AVX512VL-NEXT: vmovq %xmm1, %rbx
+; AVX512VL-NEXT: andq %rsi, %rdi
+; AVX512VL-NEXT: vmovq %xmm0, %rsi
+; AVX512VL-NEXT: andq %rbx, %rsi
+; AVX512VL-NEXT: movq %rdx, 56(%rax)
+; AVX512VL-NEXT: movq %r8, 48(%rax)
; AVX512VL-NEXT: movq %r11, 40(%rax)
; AVX512VL-NEXT: movq %r10, 32(%rax)
-; AVX512VL-NEXT: movq %r9, 8(%rax)
-; AVX512VL-NEXT: movq %r8, (%rax)
-; AVX512VL-NEXT: movq %rsi, 24(%rax)
-; AVX512VL-NEXT: movq %rdx, 16(%rax)
-; AVX512VL-NEXT: addq $8, %rsp
+; AVX512VL-NEXT: movq %rcx, 8(%rax)
+; AVX512VL-NEXT: movq %r9, (%rax)
+; AVX512VL-NEXT: movq %rdi, 24(%rax)
+; AVX512VL-NEXT: movq %rsi, 16(%rax)
; AVX512VL-NEXT: popq %rbx
-; AVX512VL-NEXT: popq %r14
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
@@ -429,7 +423,7 @@ define i512 @bext_i512(i512 %a0, i512 %idx, i512 %len) nounwind {
; AVX512VBMI-NEXT: pushq %rax
; AVX512VBMI-NEXT: movq %rdi, %rax
; AVX512VBMI-NEXT: movl {{[0-9]+}}(%rsp), %edi
-; AVX512VBMI-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
+; AVX512VBMI-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512VBMI-NEXT: movl $1, %r11d
; AVX512VBMI-NEXT: shlxq %r10, %r11, %r14
@@ -451,57 +445,56 @@ define i512 @bext_i512(i512 %a0, i512 %idx, i512 %len) nounwind {
; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm2, %xmm1
; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %r11
; AVX512VBMI-NEXT: vmovq %xmm1, %r10
-; AVX512VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VBMI-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm2, %xmm3
+; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm2, %xmm1
; AVX512VBMI-NEXT: movq %rbx, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vpextrq $1, %xmm3, %rbx
-; AVX512VBMI-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovups %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rbx
+; AVX512VBMI-NEXT: vmovdqu %xmm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vpextrq $1, %xmm2, %r9
+; AVX512VBMI-NEXT: vmovq %xmm1, %r9
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovq %xmm3, %r8
+; AVX512VBMI-NEXT: vpextrq $1, %xmm2, %r8
; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %edi, %ecx
; AVX512VBMI-NEXT: shrl $3, %edi
; AVX512VBMI-NEXT: andl $56, %edi
-; AVX512VBMI-NEXT: vmovdqu64 -128(%rsp,%rdi), %zmm0
-; AVX512VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VBMI-NEXT: valignq {{.*#+}} zmm1 = zmm0[1,2,3,4,5,6,7],zmm1[0]
+; AVX512VBMI-NEXT: vmovdqu64 -128(%rsp,%rdi), %zmm1
+; AVX512VBMI-NEXT: valignq {{.*#+}} zmm0 = zmm1[1,2,3,4,5,6,7],zmm0[0]
; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm3
-; AVX512VBMI-NEXT: vpshrdvq %zmm3, %zmm1, %zmm0
-; AVX512VBMI-NEXT: vextracti128 $1, %ymm2, %xmm1
-; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rdx
-; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm0, %xmm3
+; AVX512VBMI-NEXT: vpshrdvq %zmm3, %zmm0, %zmm1
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm2, %xmm0
+; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rdx
+; AVX512VBMI-NEXT: vextracti32x4 $3, %zmm1, %xmm3
; AVX512VBMI-NEXT: vpextrq $1, %xmm3, %rcx
; AVX512VBMI-NEXT: vmovq %xmm2, %rsi
; AVX512VBMI-NEXT: andq %r11, %rcx
; AVX512VBMI-NEXT: vmovq %xmm3, %rdi
-; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm0, %xmm2
+; AVX512VBMI-NEXT: vextracti32x4 $2, %zmm1, %xmm2
; AVX512VBMI-NEXT: vpextrq $1, %xmm2, %r11
; AVX512VBMI-NEXT: andq %r10, %rdi
; AVX512VBMI-NEXT: andq %rbx, %r11
; AVX512VBMI-NEXT: vmovq %xmm2, %r10
-; AVX512VBMI-NEXT: andq %r8, %r10
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %r8
-; AVX512VBMI-NEXT: andq %r9, %r8
-; AVX512VBMI-NEXT: vmovq %xmm0, %r9
-; AVX512VBMI-NEXT: andq %rsi, %r9
-; AVX512VBMI-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX512VBMI-NEXT: vpextrq $1, %xmm0, %rsi
-; AVX512VBMI-NEXT: vmovq %xmm1, %rbx
+; AVX512VBMI-NEXT: andq %r9, %r10
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %r9
+; AVX512VBMI-NEXT: andq %r8, %r9
+; AVX512VBMI-NEXT: vmovq %xmm1, %r8
+; AVX512VBMI-NEXT: andq %rsi, %r8
+; AVX512VBMI-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX512VBMI-NEXT: vpextrq $1, %xmm1, %rsi
+; AVX512VBMI-NEXT: vmovq %xmm0, %rbx
; AVX512VBMI-NEXT: andq %rdx, %rsi
-; AVX512VBMI-NEXT: vmovq %xmm0, %rdx
+; AVX512VBMI-NEXT: vmovq %xmm1, %rdx
; AVX512VBMI-NEXT: andq %rbx, %rdx
; AVX512VBMI-NEXT: movq %rcx, 56(%rax)
; AVX512VBMI-NEXT: movq %rdi, 48(%rax)
; AVX512VBMI-NEXT: movq %r11, 40(%rax)
; AVX512VBMI-NEXT: movq %r10, 32(%rax)
-; AVX512VBMI-NEXT: movq %r8, 8(%rax)
-; AVX512VBMI-NEXT: movq %r9, (%rax)
+; AVX512VBMI-NEXT: movq %r9, 8(%rax)
+; AVX512VBMI-NEXT: movq %r8, (%rax)
; AVX512VBMI-NEXT: movq %rsi, 24(%rax)
; AVX512VBMI-NEXT: movq %rdx, 16(%rax)
; AVX512VBMI-NEXT: addq $8, %rsp
@@ -638,10 +631,10 @@ define i512 @bext_i512_vector(<8 x i64> %v0, i512 %idx, i512 %len) nounwind {
; AVX2-NEXT: subq $152, %rsp
; AVX2-NEXT: vxorps %xmm2, %xmm2, %xmm2
; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovss {{.*#+}} xmm3 = [1,0,0,0]
; AVX2-NEXT: vmovups %ymm3, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
; AVX2-NEXT: movl %eax, %ecx
; AVX2-NEXT: andl $63, %ecx
@@ -676,10 +669,10 @@ define i512 @bext_i512_vector(<8 x i64> %v0, i512 %idx, i512 %len) nounwind {
; AVX2-NEXT: adcq $-1, %r8
; AVX2-NEXT: adcq $-1, %r11
; AVX2-NEXT: movq %r11, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: vmovups %ymm2, {{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm2, {{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm1, {{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, {{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm2, {{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm2, {{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %esi, %ecx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %esi
@@ -979,10 +972,10 @@ define i512 @bext_i512_load(ptr %p0, i512 %idx, i512 %len) nounwind {
; AVX2-NEXT: vmovups 32(%rsi), %ymm1
; AVX2-NEXT: vxorps %xmm2, %xmm2, %xmm2
; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovss {{.*#+}} xmm3 = [1,0,0,0]
; AVX2-NEXT: vmovups %ymm3, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
; AVX2-NEXT: movl %eax, %ecx
; AVX2-NEXT: andl $63, %ecx
@@ -3335,10 +3328,10 @@ define i512 @bzhi_i512(i512 %a0, i512 %idx) nounwind {
; AVX2-NEXT: pushq %rax
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rcx, %rax
-; AVX2-NEXT: vmovss {{.*#+}} xmm0 = [1,0,0,0]
+; AVX2-NEXT: vmovss {{.*#+}} xmm1 = [1,0,0,0]
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl {{[0-9]+}}(%rsp), %r10d
; AVX2-NEXT: movl %r10d, %ecx
@@ -3747,10 +3740,10 @@ define i512 @bzhi_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: vxorps %xmm2, %xmm2, %xmm2
; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovss {{.*#+}} xmm3 = [1,0,0,0]
+; AVX2-NEXT: vmovups %ymm3, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovss {{.*#+}} xmm2 = [1,0,0,0]
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %esi, %ecx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %esi
@@ -3968,10 +3961,10 @@ define i512 @bzhi_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovss {{.*#+}} xmm1 = [1,0,0,0]
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovss {{.*#+}} xmm0 = [1,0,0,0]
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %edx, %ecx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %edx
@@ -4290,11 +4283,11 @@ define i512 @isolate_msb_i512(i512 %a0, i512 %idx) nounwind {
; AVX2-NEXT: orq %r14, %r9
; AVX2-NEXT: orq %rbx, %r9
; AVX2-NEXT: cmovneq %r12, %rax
+; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %eax, %ecx
; AVX2-NEXT: andl $63, %ecx
@@ -4751,102 +4744,102 @@ define i512 @isolate_msb_i512_vector(<8 x i64> %v0, i512 %idx) nounwind {
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
+; AVX2-NEXT: vpextrq $1, %xmm0, %rsi
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
; AVX2-NEXT: vmovq %xmm2, %rax
; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT: vmovq %xmm1, %rdx
+; AVX2-NEXT: vpextrq $1, %xmm1, %r9
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vmovq %xmm2, %rdx
-; AVX2-NEXT: vpextrq $1, %xmm2, %rsi
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vmovdqu %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovaps {{.*#+}} ymm3 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm3, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrq $1, %xmm0, %r9
-; AVX2-NEXT: vmovq %xmm0, %r8
-; AVX2-NEXT: vmovq %xmm1, %r10
-; AVX2-NEXT: vpextrq $1, %xmm1, %r11
+; AVX2-NEXT: vmovq %xmm2, %r10
+; AVX2-NEXT: vpextrq $1, %xmm2, %r11
+; AVX2-NEXT: lzcntq %r11, %r8
; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %rsi, %rbx
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %rdx, %r14
-; AVX2-NEXT: addq $64, %r14
-; AVX2-NEXT: testq %rsi, %rsi
-; AVX2-NEXT: cmovneq %rbx, %r14
-; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r11, %rbx
-; AVX2-NEXT: lzcntq %r10, %r10
-; AVX2-NEXT: addq $64, %r10
+; AVX2-NEXT: lzcntq %r10, %rbx
+; AVX2-NEXT: addq $64, %rbx
; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovneq %rbx, %r10
-; AVX2-NEXT: subq $-128, %r10
-; AVX2-NEXT: orq %rsi, %rdx
-; AVX2-NEXT: cmovneq %r14, %r10
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq %rcx, %rdx
-; AVX2-NEXT: xorl %esi, %esi
-; AVX2-NEXT: lzcntq %rax, %rsi
-; AVX2-NEXT: addq $64, %rsi
-; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovneq %rdx, %rsi
-; AVX2-NEXT: xorl %edx, %edx
-; AVX2-NEXT: lzcntq %r9, %rdx
-; AVX2-NEXT: lzcntq %r8, %r8
+; AVX2-NEXT: cmovneq %r8, %rbx
+; AVX2-NEXT: xorl %r14d, %r14d
+; AVX2-NEXT: lzcntq %r9, %r14
+; AVX2-NEXT: xorl %r8d, %r8d
+; AVX2-NEXT: lzcntq %rdx, %r8
; AVX2-NEXT: addq $64, %r8
; AVX2-NEXT: testq %r9, %r9
-; AVX2-NEXT: cmovneq %rdx, %r8
+; AVX2-NEXT: cmovneq %r14, %r8
; AVX2-NEXT: subq $-128, %r8
+; AVX2-NEXT: orq %r11, %r10
+; AVX2-NEXT: cmovneq %rbx, %r8
+; AVX2-NEXT: xorl %edx, %edx
+; AVX2-NEXT: lzcntq %rcx, %rdx
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: lzcntq %rax, %r9
+; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: testq %rcx, %rcx
+; AVX2-NEXT: cmovneq %rdx, %r9
+; AVX2-NEXT: vmovq %xmm0, %rdx
+; AVX2-NEXT: xorl %r10d, %r10d
+; AVX2-NEXT: lzcntq %rsi, %r10
+; AVX2-NEXT: lzcntq %rdx, %rdx
+; AVX2-NEXT: addq $64, %rdx
+; AVX2-NEXT: testq %rsi, %rsi
+; AVX2-NEXT: cmovneq %r10, %rdx
+; AVX2-NEXT: subq $-128, %rdx
; AVX2-NEXT: orq %rcx, %rax
-; AVX2-NEXT: cmovneq %rsi, %r8
-; AVX2-NEXT: addq $256, %r8 # imm = 0x100
+; AVX2-NEXT: cmovneq %r9, %rdx
+; AVX2-NEXT: addq $256, %rdx # imm = 0x100
; AVX2-NEXT: vptest %ymm1, %ymm1
-; AVX2-NEXT: cmovneq %r10, %r8
-; AVX2-NEXT: vmovdqu %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqu %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movl %r8d, %ecx
+; AVX2-NEXT: cmovneq %r8, %rdx
+; AVX2-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl %edx, %ecx
; AVX2-NEXT: andl $63, %ecx
-; AVX2-NEXT: shrl $3, %r8d
-; AVX2-NEXT: andl $56, %r8d
-; AVX2-NEXT: movq -72(%rsp,%r8), %r11
-; AVX2-NEXT: movq -80(%rsp,%r8), %rax
-; AVX2-NEXT: movq %rax, %rdx
-; AVX2-NEXT: shrdq %cl, %r11, %rdx
-; AVX2-NEXT: movq -88(%rsp,%r8), %r10
-; AVX2-NEXT: movq %r10, %rsi
-; AVX2-NEXT: shrdq %cl, %rax, %rsi
-; AVX2-NEXT: movq -96(%rsp,%r8), %rax
+; AVX2-NEXT: shrl $3, %edx
+; AVX2-NEXT: andl $56, %edx
+; AVX2-NEXT: movq -72(%rsp,%rdx), %r11
+; AVX2-NEXT: movq -80(%rsp,%rdx), %rax
+; AVX2-NEXT: movq %rax, %rsi
+; AVX2-NEXT: shrdq %cl, %r11, %rsi
+; AVX2-NEXT: movq -88(%rsp,%rdx), %r10
+; AVX2-NEXT: movq %r10, %r8
+; AVX2-NEXT: shrdq %cl, %rax, %r8
+; AVX2-NEXT: movq -96(%rsp,%rdx), %rax
; AVX2-NEXT: movq %rax, %r9
; AVX2-NEXT: shrdq %cl, %r10, %r9
-; AVX2-NEXT: movq -104(%rsp,%r8), %r14
+; AVX2-NEXT: movq -104(%rsp,%rdx), %r14
; AVX2-NEXT: movq %r14, %r10
; AVX2-NEXT: shrdq %cl, %rax, %r10
-; AVX2-NEXT: movq -112(%rsp,%r8), %r15
+; AVX2-NEXT: movq -112(%rsp,%rdx), %r15
; AVX2-NEXT: movq %r15, %rbx
; AVX2-NEXT: shrdq %cl, %r14, %rbx
; AVX2-NEXT: movq %rdi, %rax
; AVX2-NEXT: vpor %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: movq -128(%rsp,%r8), %rdi
-; AVX2-NEXT: movq -120(%rsp,%r8), %r14
-; AVX2-NEXT: movq %r14, %r8
-; AVX2-NEXT: shrdq %cl, %r15, %r8
+; AVX2-NEXT: movq -128(%rsp,%rdx), %rdi
+; AVX2-NEXT: movq -120(%rsp,%rdx), %r14
+; AVX2-NEXT: movq %r14, %rdx
+; AVX2-NEXT: shrdq %cl, %r15, %rdx
; AVX2-NEXT: shrdq %cl, %r14, %rdi
; AVX2-NEXT: xorl %r14d, %r14d
; AVX2-NEXT: vptest %ymm0, %ymm0
; AVX2-NEXT: shrxq %rcx, %r11, %rcx
-; AVX2-NEXT: cmoveq %r14, %r8
+; AVX2-NEXT: cmoveq %r14, %rdx
; AVX2-NEXT: cmoveq %r14, %rbx
; AVX2-NEXT: cmoveq %r14, %r10
; AVX2-NEXT: cmoveq %r14, %r9
+; AVX2-NEXT: cmoveq %r14, %r8
; AVX2-NEXT: cmoveq %r14, %rsi
-; AVX2-NEXT: cmoveq %r14, %rdx
; AVX2-NEXT: cmoveq %r14, %rdi
; AVX2-NEXT: cmoveq %r14, %rcx
; AVX2-NEXT: movq %rcx, 56(%rax)
-; AVX2-NEXT: movq %rdx, 48(%rax)
-; AVX2-NEXT: movq %rsi, 40(%rax)
+; AVX2-NEXT: movq %rsi, 48(%rax)
+; AVX2-NEXT: movq %r8, 40(%rax)
; AVX2-NEXT: movq %r9, 32(%rax)
; AVX2-NEXT: movq %r10, 24(%rax)
; AVX2-NEXT: movq %rbx, 16(%rax)
-; AVX2-NEXT: movq %r8, 8(%rax)
+; AVX2-NEXT: movq %rdx, 8(%rax)
; AVX2-NEXT: movq %rdi, (%rax)
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: popq %r14
@@ -5200,56 +5193,55 @@ define i512 @isolate_msb_i512_load(ptr %p0, i512 %idx) nounwind {
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: vmovdqu 32(%rsi), %ymm1
-; AVX2-NEXT: movq 8(%rsi), %r8
; AVX2-NEXT: movq 16(%rsi), %rax
; AVX2-NEXT: movq 24(%rsi), %rcx
; AVX2-NEXT: movq 40(%rsi), %rdx
-; AVX2-NEXT: movq 48(%rsi), %r10
+; AVX2-NEXT: movq 48(%rsi), %r9
; AVX2-NEXT: vpor (%rsi), %ymm1, %ymm0
-; AVX2-NEXT: movq 56(%rsi), %r11
-; AVX2-NEXT: lzcntq %r11, %r9
+; AVX2-NEXT: movq 56(%rsi), %r10
+; AVX2-NEXT: lzcntq %r10, %r8
+; AVX2-NEXT: lzcntq %r9, %r11
+; AVX2-NEXT: addq $64, %r11
+; AVX2-NEXT: testq %r10, %r10
+; AVX2-NEXT: cmovneq %r8, %r11
; AVX2-NEXT: xorl %ebx, %ebx
-; AVX2-NEXT: lzcntq %r10, %rbx
-; AVX2-NEXT: addq $64, %rbx
-; AVX2-NEXT: testq %r11, %r11
-; AVX2-NEXT: cmovneq %r9, %rbx
-; AVX2-NEXT: xorl %r14d, %r14d
-; AVX2-NEXT: lzcntq %rdx, %r14
-; AVX2-NEXT: xorl %r9d, %r9d
-; AVX2-NEXT: lzcntq 32(%rsi), %r9
-; AVX2-NEXT: addq $64, %r9
+; AVX2-NEXT: lzcntq %rdx, %rbx
+; AVX2-NEXT: xorl %r8d, %r8d
+; AVX2-NEXT: lzcntq 32(%rsi), %r8
+; AVX2-NEXT: addq $64, %r8
; AVX2-NEXT: testq %rdx, %rdx
-; AVX2-NEXT: cmovneq %r14, %r9
-; AVX2-NEXT: subq $-128, %r9
-; AVX2-NEXT: orq %r11, %r10
-; AVX2-NEXT: cmovneq %rbx, %r9
+; AVX2-NEXT: cmovneq %rbx, %r8
+; AVX2-NEXT: subq $-128, %r8
+; AVX2-NEXT: orq %r10, %r9
+; AVX2-NEXT: cmovneq %r11, %r8
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: lzcntq %rcx, %rdx
-; AVX2-NEXT: xorl %r10d, %r10d
-; AVX2-NEXT: lzcntq %rax, %r10
-; AVX2-NEXT: addq $64, %r10
+; AVX2-NEXT: xorl %r9d, %r9d
+; AVX2-NEXT: lzcntq %rax, %r9
+; AVX2-NEXT: addq $64, %r9
; AVX2-NEXT: testq %rcx, %rcx
-; AVX2-NEXT: cmovneq %rdx, %r10
+; AVX2-NEXT: cmovneq %rdx, %r9
+; AVX2-NEXT: movq 8(%rsi), %r10
; AVX2-NEXT: xorl %r11d, %r11d
-; AVX2-NEXT: lzcntq %r8, %r11
+; AVX2-NEXT: lzcntq %r10, %r11
; AVX2-NEXT: xorl %edx, %edx
; AVX2-NEXT: lzcntq (%rsi), %rdx
; AVX2-NEXT: addq $64, %rdx
-; AVX2-NEXT: testq %r8, %r8
+; AVX2-NEXT: testq %r10, %r10
; AVX2-NEXT: cmovneq %r11, %rdx
; AVX2-NEXT: subq $-128, %rdx
; AVX2-NEXT: orq %rcx, %rax
-; AVX2-NEXT: cmovneq %r10, %rdx
+; AVX2-NEXT: cmovneq %r9, %rdx
; AVX2-NEXT: addq $256, %rdx # imm = 0x100
; AVX2-NEXT: vpor 48(%rsi), %xmm1, %xmm1
; AVX2-NEXT: vptest %xmm1, %xmm1
-; AVX2-NEXT: cmovneq %r9, %rdx
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovaps {{.*#+}} ymm2 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: cmovneq %r8, %rdx
+; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %edx, %ecx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %edx
diff --git a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
index 474be4465d9b7..0633daa200ef4 100644
--- a/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
+++ b/llvm/test/CodeGen/X86/bitcast-int-to-vector-bool-sext.ll
@@ -729,7 +729,6 @@ define <8 x i32> @PR157382(ptr %p0, ptr %p1, ptr %p2) {
; AVX2-NEXT: vpxor %ymm5, %ymm4, %ymm4
; AVX2-NEXT: vpcmpgtd %ymm3, %ymm1, %ymm1
; AVX2-NEXT: vpor %ymm1, %ymm4, %ymm1
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: vpcmpeqb %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpcmpeqd %xmm3, %xmm3, %xmm3
; AVX2-NEXT: vpxor %xmm3, %xmm2, %xmm2
diff --git a/llvm/test/CodeGen/X86/combine-icmp.ll b/llvm/test/CodeGen/X86/combine-icmp.ll
index 3b44339dc0b4a..b9cdaa04c1055 100644
--- a/llvm/test/CodeGen/X86/combine-icmp.ll
+++ b/llvm/test/CodeGen/X86/combine-icmp.ll
@@ -646,9 +646,8 @@ define i8 @concat_icmp_v8i64_v4i64(<4 x i64> %a0, <4 x i64> %a1) {
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm3
; AVX2-NEXT: vpackssdw %xmm3, %xmm0, %xmm0
; AVX2-NEXT: vpcmpeqq %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vpackssdw %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT: vpackssdw %xmm3, %xmm1, %xmm1
; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3],xmm1[4],xmm2[5],xmm1[6],xmm2[7]
; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3],xmm0[4],xmm2[5],xmm0[6],xmm2[7]
; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/combine-pmuldq.ll b/llvm/test/CodeGen/X86/combine-pmuldq.ll
index 20d42192b993a..a32a2648c8d90 100644
--- a/llvm/test/CodeGen/X86/combine-pmuldq.ll
+++ b/llvm/test/CodeGen/X86/combine-pmuldq.ll
@@ -447,9 +447,9 @@ define <8 x i32> @PR49658_zext(ptr %ptr, i32 %mul) {
; AVX2-LABEL: PR49658_zext:
; AVX2: # %bb.0: # %start
; AVX2-NEXT: movl %esi, %eax
-; AVX2-NEXT: vmovq %rax, %xmm0
-; AVX2-NEXT: vpbroadcastq %xmm0, %ymm1
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovq %rax, %xmm1
+; AVX2-NEXT: vpbroadcastq %xmm1, %ymm1
; AVX2-NEXT: movq $-2097152, %rax # imm = 0xFFE00000
; AVX2-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,2,0,2,4,6,4,6]
; AVX2-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
@@ -472,8 +472,8 @@ define <8 x i32> @PR49658_zext(ptr %ptr, i32 %mul) {
; AVX512VL-LABEL: PR49658_zext:
; AVX512VL: # %bb.0: # %start
; AVX512VL-NEXT: movl %esi, %eax
-; AVX512VL-NEXT: vpbroadcastq %rax, %zmm1
; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vpbroadcastq %rax, %zmm1
; AVX512VL-NEXT: movq $-2097152, %rax # imm = 0xFFE00000
; AVX512VL-NEXT: vpmovqd %zmm1, %ymm1
; AVX512VL-NEXT: vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
@@ -495,8 +495,8 @@ define <8 x i32> @PR49658_zext(ptr %ptr, i32 %mul) {
; AVX512DQVL-LABEL: PR49658_zext:
; AVX512DQVL: # %bb.0: # %start
; AVX512DQVL-NEXT: movl %esi, %eax
-; AVX512DQVL-NEXT: vpbroadcastq %rax, %zmm1
; AVX512DQVL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQVL-NEXT: vpbroadcastq %rax, %zmm1
; AVX512DQVL-NEXT: movq $-2097152, %rax # imm = 0xFFE00000
; AVX512DQVL-NEXT: vpmovqd %zmm1, %ymm1
; AVX512DQVL-NEXT: vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
@@ -639,9 +639,9 @@ define <8 x i32> @PR49658_sext(ptr %ptr, i32 %mul) {
; AVX2-LABEL: PR49658_sext:
; AVX2: # %bb.0: # %start
; AVX2-NEXT: movslq %esi, %rax
-; AVX2-NEXT: vmovq %rax, %xmm0
-; AVX2-NEXT: vpbroadcastq %xmm0, %ymm1
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovq %rax, %xmm1
+; AVX2-NEXT: vpbroadcastq %xmm1, %ymm1
; AVX2-NEXT: movq $-2097152, %rax # imm = 0xFFE00000
; AVX2-NEXT: vshufps {{.*#+}} ymm1 = ymm1[0,2,0,2,4,6,4,6]
; AVX2-NEXT: vpermpd {{.*#+}} ymm1 = ymm1[0,2,1,3]
@@ -664,8 +664,8 @@ define <8 x i32> @PR49658_sext(ptr %ptr, i32 %mul) {
; AVX512VL-LABEL: PR49658_sext:
; AVX512VL: # %bb.0: # %start
; AVX512VL-NEXT: movslq %esi, %rax
-; AVX512VL-NEXT: vpbroadcastq %rax, %zmm1
; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vpbroadcastq %rax, %zmm1
; AVX512VL-NEXT: movq $-2097152, %rax # imm = 0xFFE00000
; AVX512VL-NEXT: vpmovqd %zmm1, %ymm1
; AVX512VL-NEXT: vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
@@ -687,8 +687,8 @@ define <8 x i32> @PR49658_sext(ptr %ptr, i32 %mul) {
; AVX512DQVL-LABEL: PR49658_sext:
; AVX512DQVL: # %bb.0: # %start
; AVX512DQVL-NEXT: movslq %esi, %rax
-; AVX512DQVL-NEXT: vpbroadcastq %rax, %zmm1
; AVX512DQVL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQVL-NEXT: vpbroadcastq %rax, %zmm1
; AVX512DQVL-NEXT: movq $-2097152, %rax # imm = 0xFFE00000
; AVX512DQVL-NEXT: vpmovqd %zmm1, %ymm1
; AVX512DQVL-NEXT: vpshufd {{.*#+}} ymm2 = ymm1[1,1,3,3,5,5,7,7]
diff --git a/llvm/test/CodeGen/X86/combine-storetomstore.ll b/llvm/test/CodeGen/X86/combine-storetomstore.ll
index 45a1172b2323e..a87047761b691 100644
--- a/llvm/test/CodeGen/X86/combine-storetomstore.ll
+++ b/llvm/test/CodeGen/X86/combine-storetomstore.ll
@@ -1195,6 +1195,7 @@ define void @test_masked_store_intervening(<8 x i32> %x, ptr %ptr, <8 x i1> %mas
; AVX-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill
; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX-NEXT: vmovaps %ymm0, (%rdi)
+; AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX-NEXT: callq use_vec at PLT
; AVX-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload
; AVX-NEXT: vmovaps %ymm0, (%rbx)
@@ -1215,6 +1216,7 @@ define void @test_masked_store_intervening(<8 x i32> %x, ptr %ptr, <8 x i1> %mas
; AVX2-NEXT: vmovups %ymm0, (%rsp) # 32-byte Spill
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovaps %ymm0, (%rdi)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: callq use_vec at PLT
; AVX2-NEXT: vmovups (%rsp), %ymm0 # 32-byte Reload
; AVX2-NEXT: vmovaps %ymm0, (%rbx)
@@ -1236,6 +1238,7 @@ define void @test_masked_store_intervening(<8 x i32> %x, ptr %ptr, <8 x i1> %mas
; AVX512-NEXT: vmovups %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512-NEXT: vmovaps %ymm0, (%rdi)
+; AVX512-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512-NEXT: callq use_vec at PLT
; AVX512-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
; AVX512-NEXT: kmovw {{[-0-9]+}}(%r{{[sb]}}p), %k1 # 2-byte Reload
diff --git a/llvm/test/CodeGen/X86/dpbusd.ll b/llvm/test/CodeGen/X86/dpbusd.ll
index 91c32221cfe75..077a82f917759 100644
--- a/llvm/test/CodeGen/X86/dpbusd.ll
+++ b/llvm/test/CodeGen/X86/dpbusd.ll
@@ -299,12 +299,12 @@ define i32 @vpdpbusd_256(ptr%a, ptr%b, i32 %c, i32 %n) {
;
; AVX512VNNI-LABEL: vpdpbusd_256:
; AVX512VNNI: # %bb.0: # %entry
-; AVX512VNNI-NEXT: vmovq {{.*#+}} xmm0 = mem[0],zero
+; AVX512VNNI-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512VNNI-NEXT: vmovq {{.*#+}} xmm1 = mem[0],zero
-; AVX512VNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VNNI-NEXT: vpdpbusd %zmm0, %zmm1, %zmm2
-; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
-; AVX512VNNI-NEXT: vpaddd %xmm0, %xmm2, %xmm0
+; AVX512VNNI-NEXT: vmovq {{.*#+}} xmm2 = mem[0],zero
+; AVX512VNNI-NEXT: vpdpbusd %zmm1, %zmm2, %zmm0
+; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512VNNI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512VNNI-NEXT: vmovd %xmm0, %eax
; AVX512VNNI-NEXT: addl %edx, %eax
; AVX512VNNI-NEXT: vzeroupper
@@ -347,11 +347,11 @@ define i32 @vpdpbusd_128(ptr%a, ptr%b, i32 %c, i32 %n) {
;
; AVX512VNNI-LABEL: vpdpbusd_128:
; AVX512VNNI: # %bb.0: # %entry
-; AVX512VNNI-NEXT: vmovd {{.*#+}} xmm0 = mem[0],zero,zero,zero
+; AVX512VNNI-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512VNNI-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; AVX512VNNI-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VNNI-NEXT: vpdpbusd %zmm0, %zmm1, %zmm2
-; AVX512VNNI-NEXT: vmovd %xmm2, %eax
+; AVX512VNNI-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
+; AVX512VNNI-NEXT: vpdpbusd %zmm1, %zmm2, %zmm0
+; AVX512VNNI-NEXT: vmovd %xmm0, %eax
; AVX512VNNI-NEXT: addl %edx, %eax
; AVX512VNNI-NEXT: vzeroupper
; AVX512VNNI-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/dpbusd_const.ll b/llvm/test/CodeGen/X86/dpbusd_const.ll
index 1d6c3f7c5c6e8..852e3931be4d1 100644
--- a/llvm/test/CodeGen/X86/dpbusd_const.ll
+++ b/llvm/test/CodeGen/X86/dpbusd_const.ll
@@ -211,8 +211,8 @@ define i32 @mul_16xi8_zc(<16 x i8> %a, i32 %c) {
;
; AVX512VNNI-LABEL: mul_16xi8_zc:
; AVX512VNNI: # %bb.0: # %entry
-; AVX512VNNI-NEXT: vmovdqa %xmm0, %xmm0
; AVX512VNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512VNNI-NEXT: vmovdqa %xmm0, %xmm0
; AVX512VNNI-NEXT: vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
; AVX512VNNI-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; AVX512VNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm0
@@ -260,8 +260,8 @@ define i32 @mul_32xi8_zc(<32 x i8> %a, i32 %c) {
;
; AVX512VNNI-LABEL: mul_32xi8_zc:
; AVX512VNNI: # %bb.0: # %entry
-; AVX512VNNI-NEXT: vmovdqa %ymm0, %ymm0
; AVX512VNNI-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512VNNI-NEXT: vmovdqa %ymm0, %ymm0
; AVX512VNNI-NEXT: vpdpbusd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm1
; AVX512VNNI-NEXT: vextracti128 $1, %ymm1, %xmm0
; AVX512VNNI-NEXT: vpaddd %xmm0, %xmm1, %xmm0
@@ -299,12 +299,12 @@ entry:
define i32 @mul_64xi8_zc(<64 x i8> %a, i32 %c) {
; AVXVNNI-AVX-LABEL: mul_64xi8_zc:
; AVXVNNI-AVX: # %bb.0: # %entry
-; AVXVNNI-AVX-NEXT: vpbroadcastd {{.*#+}} ymm2 = [0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64]
-; AVXVNNI-AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVXVNNI-AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVXVNNI-AVX-NEXT: vpbroadcastd {{.*#+}} ymm3 = [0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64]
; AVXVNNI-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVXVNNI-AVX-NEXT: {vex} vpdpbusd %ymm2, %ymm1, %ymm4
-; AVXVNNI-AVX-NEXT: {vex} vpdpbusd %ymm2, %ymm0, %ymm3
-; AVXVNNI-AVX-NEXT: vpaddd %ymm4, %ymm3, %ymm0
+; AVXVNNI-AVX-NEXT: {vex} vpdpbusd %ymm3, %ymm1, %ymm4
+; AVXVNNI-AVX-NEXT: {vex} vpdpbusd %ymm3, %ymm0, %ymm2
+; AVXVNNI-AVX-NEXT: vpaddd %ymm4, %ymm2, %ymm0
; AVXVNNI-AVX-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVXVNNI-AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVXVNNI-AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -319,12 +319,12 @@ define i32 @mul_64xi8_zc(<64 x i8> %a, i32 %c) {
; AVXVNNI-AVX512-LABEL: mul_64xi8_zc:
; AVXVNNI-AVX512: # %bb.0: # %entry
; AVXVNNI-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVXVNNI-AVX512-NEXT: vpbroadcastd {{.*#+}} ymm2 = [0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64]
-; AVXVNNI-AVX512-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVXVNNI-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVXVNNI-AVX512-NEXT: vpbroadcastd {{.*#+}} ymm3 = [0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64,0,1,2,64]
; AVXVNNI-AVX512-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd %ymm2, %ymm1, %ymm4
-; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd %ymm2, %ymm0, %ymm3
-; AVXVNNI-AVX512-NEXT: vpaddd %ymm4, %ymm3, %ymm0
+; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd %ymm3, %ymm1, %ymm4
+; AVXVNNI-AVX512-NEXT: {vex} vpdpbusd %ymm3, %ymm0, %ymm2
+; AVXVNNI-AVX512-NEXT: vpaddd %ymm4, %ymm2, %ymm0
; AVXVNNI-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVXVNNI-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
index 2026046917ea0..7527a4b9c8d52 100644
--- a/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
+++ b/llvm/test/CodeGen/X86/gfni-funnel-shifts.ll
@@ -1616,8 +1616,8 @@ define <64 x i8> @var_fshl_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
; GFNIAVX512BW-LABEL: var_fshl_v64i8:
; GFNIAVX512BW: # %bb.0:
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; GFNIAVX512BW-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]
; GFNIAVX512BW-NEXT: vpsllvw %zmm5, %zmm3, %zmm3
; GFNIAVX512BW-NEXT: vpsrlw $8, %zmm3, %zmm3
@@ -1858,8 +1858,8 @@ define <64 x i8> @var_fshr_v64i8(<64 x i8> %a, <64 x i8> %b, <64 x i8> %amt) nou
; GFNIAVX512BW-LABEL: var_fshr_v64i8:
; GFNIAVX512BW: # %bb.0:
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; GFNIAVX512BW-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]
; GFNIAVX512BW-NEXT: vpsrlvw %zmm5, %zmm3, %zmm3
; GFNIAVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
diff --git a/llvm/test/CodeGen/X86/gfni-rotates.ll b/llvm/test/CodeGen/X86/gfni-rotates.ll
index fe14fb05ebb9e..fad7a384171e3 100644
--- a/llvm/test/CodeGen/X86/gfni-rotates.ll
+++ b/llvm/test/CodeGen/X86/gfni-rotates.ll
@@ -480,8 +480,8 @@ define <32 x i8> @var_rotl_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
;
; GFNIAVX512BW-LABEL: var_rotl_v32i8:
; GFNIAVX512BW: # %bb.0:
-; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; GFNIAVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; GFNIAVX512BW-NEXT: vpsllvw %ymm3, %ymm4, %ymm3
@@ -595,8 +595,8 @@ define <32 x i8> @var_rotr_v32i8(<32 x i8> %a, <32 x i8> %amt) nounwind {
;
; GFNIAVX512BW-LABEL: var_rotr_v32i8:
; GFNIAVX512BW: # %bb.0:
-; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; GFNIAVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; GFNIAVX512BW-NEXT: vpsrlvw %ymm3, %ymm4, %ymm3
@@ -1159,8 +1159,8 @@ define <64 x i8> @var_rotl_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
;
; GFNIAVX512BW-LABEL: var_rotl_v64i8:
; GFNIAVX512BW: # %bb.0:
-; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; GFNIAVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; GFNIAVX512BW-NEXT: vpsllvw %zmm3, %zmm4, %zmm3
@@ -1352,8 +1352,8 @@ define <64 x i8> @var_rotr_v64i8(<64 x i8> %a, <64 x i8> %amt) nounwind {
;
; GFNIAVX512BW-LABEL: var_rotr_v64i8:
; GFNIAVX512BW: # %bb.0:
-; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; GFNIAVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; GFNIAVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; GFNIAVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; GFNIAVX512BW-NEXT: vpsrlvw %zmm3, %zmm4, %zmm3
diff --git a/llvm/test/CodeGen/X86/insert-subvector-broadcast.ll b/llvm/test/CodeGen/X86/insert-subvector-broadcast.ll
index 0717a01ac2abc..33e42475d0348 100644
--- a/llvm/test/CodeGen/X86/insert-subvector-broadcast.ll
+++ b/llvm/test/CodeGen/X86/insert-subvector-broadcast.ll
@@ -6,11 +6,11 @@ define void @insert_subvector_broadcast_as_blend() {
; CHECK: # %bb.0:
; CHECK-NEXT: movq (%rax), %rax
; CHECK-NEXT: incq %rax
-; CHECK-NEXT: vpbroadcastq %rax, %zmm0
-; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; CHECK-NEXT: valignq {{.*#+}} zmm1 = zmm1[7],zmm0[0,1,2,3,4,5,6]
-; CHECK-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %k0
-; CHECK-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k1
+; CHECK-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; CHECK-NEXT: vpbroadcastq %rax, %zmm1
+; CHECK-NEXT: valignq {{.*#+}} zmm0 = zmm0[7],zmm1[0,1,2,3,4,5,6]
+; CHECK-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %k0
+; CHECK-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %k1
; CHECK-NEXT: kunpckbw %k0, %k1, %k1
; CHECK-NEXT: vmovdqu8 {{.*#+}} xmm0 {%k1} {z} = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; CHECK-NEXT: vmovdqa %xmm0, (%rax)
diff --git a/llvm/test/CodeGen/X86/kshift.ll b/llvm/test/CodeGen/X86/kshift.ll
index b790a0c3d74d6..f37af76b45abc 100644
--- a/llvm/test/CodeGen/X86/kshift.ll
+++ b/llvm/test/CodeGen/X86/kshift.ll
@@ -61,23 +61,23 @@ define i16 @kshiftl_v16i1_1(<16 x i32> %x, <16 x i32> %y) {
define i32 @kshiftl_v32i1_1(<32 x i16> %x, <32 x i16> %y) {
; KNL-LABEL: kshiftl_v32i1_1:
; KNL: # %bb.0:
-; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; KNL-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; KNL-NEXT: vpcmpeqw %ymm3, %ymm2, %ymm2
-; KNL-NEXT: vpmovsxwd %ymm2, %zmm2
-; KNL-NEXT: vptestmd %zmm2, %zmm2, %k1
-; KNL-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm0
+; KNL-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; KNL-NEXT: vpcmpeqw %ymm2, %ymm3, %ymm3
+; KNL-NEXT: vpmovsxwd %ymm3, %zmm3
+; KNL-NEXT: vptestmd %zmm3, %zmm3, %k1
+; KNL-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
; KNL-NEXT: vpmovsxwd %ymm0, %zmm0
; KNL-NEXT: vptestmd %zmm0, %zmm0, %k2
; KNL-NEXT: vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1
-; KNL-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1
-; KNL-NEXT: valignd {{.*#+}} zmm0 = zmm0[15],zmm2[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14]
+; KNL-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1
+; KNL-NEXT: valignd {{.*#+}} zmm0 = zmm0[15],zmm3[0,1,2,3,4,5,6,7,8,9,10,11,12,13,14]
; KNL-NEXT: kshiftlw $1, %k2, %k1
-; KNL-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; KNL-NEXT: vpcmpeqw %ymm3, %ymm2, %ymm2
-; KNL-NEXT: vpmovsxwd %ymm2, %zmm2
-; KNL-NEXT: vptestmd %zmm2, %zmm2, %k2
-; KNL-NEXT: vpcmpeqw %ymm3, %ymm1, %ymm1
+; KNL-NEXT: vextracti64x4 $1, %zmm1, %ymm3
+; KNL-NEXT: vpcmpeqw %ymm2, %ymm3, %ymm3
+; KNL-NEXT: vpmovsxwd %ymm3, %zmm3
+; KNL-NEXT: vptestmd %zmm3, %zmm3, %k2
+; KNL-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm1
; KNL-NEXT: vpmovsxwd %ymm1, %zmm1
; KNL-NEXT: vptestmd %zmm1, %zmm1, %k0 {%k1}
; KNL-NEXT: kmovw %k0, %ecx
@@ -357,23 +357,23 @@ define i16 @kshiftr_v16i1_1(<16 x i32> %x, <16 x i32> %y) {
define i32 @kshiftr_v32i1_1(<32 x i16> %x, <32 x i16> %y) {
; KNL-LABEL: kshiftr_v32i1_1:
; KNL: # %bb.0:
-; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm2
-; KNL-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; KNL-NEXT: vpcmpeqw %ymm3, %ymm2, %ymm2
-; KNL-NEXT: vpmovsxwd %ymm2, %zmm2
-; KNL-NEXT: vptestmd %zmm2, %zmm2, %k1
-; KNL-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm0
+; KNL-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm3
+; KNL-NEXT: vpcmpeqw %ymm2, %ymm3, %ymm3
+; KNL-NEXT: vpmovsxwd %ymm3, %zmm3
+; KNL-NEXT: vptestmd %zmm3, %zmm3, %k1
+; KNL-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
; KNL-NEXT: vpmovsxwd %ymm0, %zmm0
; KNL-NEXT: vptestmd %zmm0, %zmm0, %k2
; KNL-NEXT: vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1
-; KNL-NEXT: vpternlogd {{.*#+}} zmm2 {%k1} {z} = -1
-; KNL-NEXT: valignd {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm2[0]
+; KNL-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1
+; KNL-NEXT: valignd {{.*#+}} zmm0 = zmm0[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],zmm3[0]
; KNL-NEXT: kshiftrw $1, %k1, %k1
-; KNL-NEXT: vpcmpeqw %ymm3, %ymm1, %ymm2
-; KNL-NEXT: vpmovsxwd %ymm2, %zmm2
-; KNL-NEXT: vptestmd %zmm2, %zmm2, %k2
+; KNL-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm3
+; KNL-NEXT: vpmovsxwd %ymm3, %zmm3
+; KNL-NEXT: vptestmd %zmm3, %zmm3, %k2
; KNL-NEXT: vextracti64x4 $1, %zmm1, %ymm1
-; KNL-NEXT: vpcmpeqw %ymm3, %ymm1, %ymm1
+; KNL-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm1
; KNL-NEXT: vpmovsxwd %ymm1, %zmm1
; KNL-NEXT: vptestmd %zmm1, %zmm1, %k0 {%k1}
; KNL-NEXT: kmovw %k0, %ecx
@@ -403,8 +403,8 @@ define i32 @kshiftr_v32i1_1(<32 x i16> %x, <32 x i16> %y) {
define i64 @kshiftr_v64i1_1(<64 x i8> %x, <64 x i8> %y) {
; KNL-LABEL: kshiftr_v64i1_1:
; KNL: # %bb.0:
-; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm3
; KNL-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm3
; KNL-NEXT: vpcmpeqb %ymm2, %ymm3, %ymm3
; KNL-NEXT: vextracti128 $1, %ymm3, %xmm4
; KNL-NEXT: vpmovsxbd %xmm4, %zmm4
@@ -529,8 +529,8 @@ define i16 @kshiftr_v16i1_15(<16 x i32> %x, <16 x i32> %y) {
define i32 @kshiftr_v32i1_31(<32 x i16> %x, <32 x i16> %y) {
; KNL-LABEL: kshiftr_v32i1_31:
; KNL: # %bb.0:
-; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; KNL-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; KNL-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; KNL-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
; KNL-NEXT: vpmovsxwd %ymm0, %zmm0
; KNL-NEXT: vptestmd %zmm0, %zmm0, %k0
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 4ee77c0bdf70c..417281b08d289 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -132,8 +132,8 @@ define i32 @_Z10test_shortPsS_i_256(ptr nocapture readonly, ptr nocapture readon
;
; AVX1-LABEL: _Z10test_shortPsS_i_256:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: xorl %ecx, %ecx
; AVX1-NEXT: .p2align 4
; AVX1-NEXT: .LBB1_1: # %vector.body
@@ -158,8 +158,8 @@ define i32 @_Z10test_shortPsS_i_256(ptr nocapture readonly, ptr nocapture readon
;
; AVX256-LABEL: _Z10test_shortPsS_i_256:
; AVX256: # %bb.0: # %entry
-; AVX256-NEXT: movl %edx, %eax
; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX256-NEXT: movl %edx, %eax
; AVX256-NEXT: xorl %ecx, %ecx
; AVX256-NEXT: .p2align 4
; AVX256-NEXT: .LBB1_1: # %vector.body
@@ -241,8 +241,8 @@ define i32 @_Z10test_shortPsS_i_512(ptr nocapture readonly, ptr nocapture readon
;
; AVX1-LABEL: _Z10test_shortPsS_i_512:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: xorl %ecx, %ecx
; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX1-NEXT: .p2align 4
@@ -275,8 +275,8 @@ define i32 @_Z10test_shortPsS_i_512(ptr nocapture readonly, ptr nocapture readon
;
; AVX2-LABEL: _Z10test_shortPsS_i_512:
; AVX2: # %bb.0: # %entry
-; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
@@ -302,8 +302,8 @@ define i32 @_Z10test_shortPsS_i_512(ptr nocapture readonly, ptr nocapture readon
;
; AVX512-LABEL: _Z10test_shortPsS_i_512:
; AVX512: # %bb.0: # %entry
-; AVX512-NEXT: movl %edx, %eax
; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: movl %edx, %eax
; AVX512-NEXT: xorl %ecx, %ecx
; AVX512-NEXT: .p2align 4
; AVX512-NEXT: .LBB2_1: # %vector.body
@@ -401,8 +401,8 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
;
; AVX1-LABEL: _Z10test_shortPsS_i_1024:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: xorl %ecx, %ecx
; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
@@ -449,8 +449,8 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
;
; AVX2-LABEL: _Z10test_shortPsS_i_1024:
; AVX2: # %bb.0: # %entry
-; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
@@ -482,8 +482,8 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
;
; AVX512F-LABEL: _Z10test_shortPsS_i_1024:
; AVX512F: # %bb.0: # %entry
-; AVX512F-NEXT: movl %edx, %eax
; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: movl %edx, %eax
; AVX512F-NEXT: xorl %ecx, %ecx
; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512F-NEXT: .p2align 4
@@ -514,8 +514,8 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
;
; AVX512BW-LABEL: _Z10test_shortPsS_i_1024:
; AVX512BW: # %bb.0: # %entry
-; AVX512BW-NEXT: movl %edx, %eax
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movl %edx, %eax
; AVX512BW-NEXT: xorl %ecx, %ecx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: .p2align 4
@@ -724,8 +724,8 @@ define i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocapture readonly
;
; AVX1-LABEL: _Z9test_charPcS_i_256:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: xorl %ecx, %ecx
; AVX1-NEXT: .p2align 4
; AVX1-NEXT: .LBB5_1: # %vector.body
@@ -751,8 +751,8 @@ define i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocapture readonly
;
; AVX256-LABEL: _Z9test_charPcS_i_256:
; AVX256: # %bb.0: # %entry
-; AVX256-NEXT: movl %edx, %eax
; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX256-NEXT: movl %edx, %eax
; AVX256-NEXT: xorl %ecx, %ecx
; AVX256-NEXT: .p2align 4
; AVX256-NEXT: .LBB5_1: # %vector.body
@@ -873,8 +873,8 @@ define i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocapture readonly
;
; AVX1-LABEL: _Z9test_charPcS_i_512:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: xorl %ecx, %ecx
; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX1-NEXT: .p2align 4
@@ -909,8 +909,8 @@ define i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocapture readonly
;
; AVX2-LABEL: _Z9test_charPcS_i_512:
; AVX2: # %bb.0: # %entry
-; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
@@ -937,8 +937,8 @@ define i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocapture readonly
;
; AVX512-LABEL: _Z9test_charPcS_i_512:
; AVX512: # %bb.0: # %entry
-; AVX512-NEXT: movl %edx, %eax
; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: movl %edx, %eax
; AVX512-NEXT: xorl %ecx, %ecx
; AVX512-NEXT: .p2align 4
; AVX512-NEXT: .LBB6_1: # %vector.body
@@ -1095,8 +1095,8 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
;
; AVX1-LABEL: _Z9test_charPcS_i_1024:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: xorl %ecx, %ecx
; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
@@ -1147,8 +1147,8 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
;
; AVX2-LABEL: _Z9test_charPcS_i_1024:
; AVX2: # %bb.0: # %entry
-; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
@@ -1182,8 +1182,8 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
;
; AVX512F-LABEL: _Z9test_charPcS_i_1024:
; AVX512F: # %bb.0: # %entry
-; AVX512F-NEXT: movl %edx, %eax
; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: movl %edx, %eax
; AVX512F-NEXT: xorl %ecx, %ecx
; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512F-NEXT: .p2align 4
@@ -1216,8 +1216,8 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
;
; AVX512BW-LABEL: _Z9test_charPcS_i_1024:
; AVX512BW: # %bb.0: # %entry
-; AVX512BW-NEXT: movl %edx, %eax
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movl %edx, %eax
; AVX512BW-NEXT: xorl %ecx, %ecx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: .p2align 4
@@ -1430,8 +1430,8 @@ define i32 @test_unsigned_short_256(ptr nocapture readonly, ptr nocapture readon
;
; AVX1-LABEL: test_unsigned_short_256:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: xorl %ecx, %ecx
; AVX1-NEXT: .p2align 4
; AVX1-NEXT: .LBB9_1: # %vector.body
@@ -1462,8 +1462,8 @@ define i32 @test_unsigned_short_256(ptr nocapture readonly, ptr nocapture readon
;
; AVX256-LABEL: test_unsigned_short_256:
; AVX256: # %bb.0: # %entry
-; AVX256-NEXT: movl %edx, %eax
; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX256-NEXT: movl %edx, %eax
; AVX256-NEXT: xorl %ecx, %ecx
; AVX256-NEXT: .p2align 4
; AVX256-NEXT: .LBB9_1: # %vector.body
@@ -1600,8 +1600,8 @@ define i32 @test_unsigned_short_512(ptr nocapture readonly, ptr nocapture readon
;
; AVX1-LABEL: test_unsigned_short_512:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: xorl %ecx, %ecx
; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX1-NEXT: .p2align 4
@@ -1646,8 +1646,8 @@ define i32 @test_unsigned_short_512(ptr nocapture readonly, ptr nocapture readon
;
; AVX2-LABEL: test_unsigned_short_512:
; AVX2: # %bb.0: # %entry
-; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
@@ -1678,8 +1678,8 @@ define i32 @test_unsigned_short_512(ptr nocapture readonly, ptr nocapture readon
;
; AVX512-LABEL: test_unsigned_short_512:
; AVX512: # %bb.0: # %entry
-; AVX512-NEXT: movl %edx, %eax
; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: movl %edx, %eax
; AVX512-NEXT: xorl %ecx, %ecx
; AVX512-NEXT: .p2align 4
; AVX512-NEXT: .LBB10_1: # %vector.body
@@ -1870,8 +1870,8 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
;
; AVX1-LABEL: test_unsigned_short_1024:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: xorl %ecx, %ecx
; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
@@ -1944,8 +1944,8 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
;
; AVX2-LABEL: test_unsigned_short_1024:
; AVX2: # %bb.0: # %entry
-; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movl %edx, %eax
; AVX2-NEXT: xorl %ecx, %ecx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
@@ -1988,8 +1988,8 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
;
; AVX512-LABEL: test_unsigned_short_1024:
; AVX512: # %bb.0: # %entry
-; AVX512-NEXT: movl %edx, %eax
; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512-NEXT: movl %edx, %eax
; AVX512-NEXT: xorl %ecx, %ecx
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: .p2align 4
@@ -3061,8 +3061,8 @@ define i64 @sum_and_sum_of_squares(ptr %a, i32 %n) {
;
; AVX1-LABEL: sum_and_sum_of_squares:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: movl %esi, %eax
; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: movl %esi, %eax
; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX1-NEXT: .p2align 4
; AVX1-NEXT: .LBB35_1: # %vector.body
@@ -3104,8 +3104,8 @@ define i64 @sum_and_sum_of_squares(ptr %a, i32 %n) {
;
; AVX256-LABEL: sum_and_sum_of_squares:
; AVX256: # %bb.0: # %entry
-; AVX256-NEXT: movl %esi, %eax
; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX256-NEXT: movl %esi, %eax
; AVX256-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX256-NEXT: .p2align 4
; AVX256-NEXT: .LBB35_1: # %vector.body
@@ -3222,8 +3222,8 @@ define i32 @sum_of_square_differences(ptr %a, ptr %b, i32 %n) {
;
; AVX1-LABEL: sum_of_square_differences:
; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: movl %edx, %eax
; AVX1-NEXT: xorl %ecx, %ecx
; AVX1-NEXT: .p2align 4
; AVX1-NEXT: .LBB36_1: # %vector.body
@@ -3250,8 +3250,8 @@ define i32 @sum_of_square_differences(ptr %a, ptr %b, i32 %n) {
;
; AVX256-LABEL: sum_of_square_differences:
; AVX256: # %bb.0: # %entry
-; AVX256-NEXT: movl %edx, %eax
; AVX256-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX256-NEXT: movl %edx, %eax
; AVX256-NEXT: xorl %ecx, %ecx
; AVX256-NEXT: .p2align 4
; AVX256-NEXT: .LBB36_1: # %vector.body
diff --git a/llvm/test/CodeGen/X86/masked_gather.ll b/llvm/test/CodeGen/X86/masked_gather.ll
index d55c09cf4a033..34d9b5903ac9c 100644
--- a/llvm/test/CodeGen/X86/masked_gather.ll
+++ b/llvm/test/CodeGen/X86/masked_gather.ll
@@ -2579,8 +2579,8 @@ define <8 x i32> @masked_gather_v8i32_v8i32(i8 %trigger) {
; AVX512F-NEXT: movzbl %dil, %eax
; AVX512F-NEXT: kmovw %eax, %k1
; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512F-NEXT: kmovw %k1, %k2
; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: kmovw %k1, %k2
; AVX512F-NEXT: vpgatherdd c+12(,%zmm0), %zmm1 {%k2}
; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512F-NEXT: vpgatherdd c+28(,%zmm0), %zmm2 {%k1}
diff --git a/llvm/test/CodeGen/X86/masked_gather_scatter.ll b/llvm/test/CodeGen/X86/masked_gather_scatter.ll
index ce4c5c14110a6..18ba9c6b91c86 100644
--- a/llvm/test/CodeGen/X86/masked_gather_scatter.ll
+++ b/llvm/test/CodeGen/X86/masked_gather_scatter.ll
@@ -19,8 +19,8 @@
define <16 x float> @test1(ptr %base, <16 x i32> %ind) {
; X64-LABEL: test1:
; X64: # %bb.0:
-; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}
; X64-NEXT: vmovaps %zmm1, %zmm0
; X64-NEXT: retq
@@ -28,8 +28,8 @@ define <16 x float> @test1(ptr %base, <16 x i32> %ind) {
; X86-LABEL: test1:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vgatherdps (%eax,%zmm0,4), %zmm1 {%k1}
; X86-NEXT: vmovaps %zmm1, %zmm0
; X86-NEXT: retl
@@ -233,8 +233,8 @@ define <8 x i32> @test6(<8 x i32>%a1, <8 x ptr> %ptr) {
;
; X64-SKX-LABEL: test6:
; X64-SKX: # %bb.0:
-; X64-SKX-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; X64-SKX-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-NEXT: kxnorb %k0, %k0, %k2
; X64-SKX-NEXT: vpgatherqd (,%zmm1), %ymm2 {%k2}
; X64-SKX-NEXT: vpscatterqd %ymm0, (,%zmm1) {%k1}
@@ -243,8 +243,8 @@ define <8 x i32> @test6(<8 x i32>%a1, <8 x ptr> %ptr) {
;
; X86-SKX-LABEL: test6:
; X86-SKX: # %bb.0:
-; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: kxnorb %k0, %k0, %k2
; X86-SKX-NEXT: vpgatherdd (,%ymm1), %ymm2 {%k2}
; X86-SKX-NEXT: vpscatterdd %ymm0, (,%ymm1) {%k1}
@@ -405,8 +405,8 @@ define <8 x i32> @test9(ptr %base, <8 x i64> %ind1, <8 x i32>%ind5) {
; X64-SKX-SMALL-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero
; X64-SKX-SMALL-NEXT: vpmuldq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm1
; X64-SKX-SMALL-NEXT: vpaddq %zmm1, %zmm0, %zmm1
-; X64-SKX-SMALL-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-SMALL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-SKX-SMALL-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-SMALL-NEXT: vpgatherqd 72(,%zmm1), %ymm0 {%k1}
; X64-SKX-SMALL-NEXT: retq
;
@@ -420,8 +420,8 @@ define <8 x i32> @test9(ptr %base, <8 x i64> %ind1, <8 x i32>%ind5) {
; X64-SKX-LARGE-NEXT: vpmullq (%rax){1to8}, %zmm0, %zmm0
; X64-SKX-LARGE-NEXT: vpaddq %zmm0, %zmm2, %zmm0
; X64-SKX-LARGE-NEXT: vpaddq %zmm1, %zmm0, %zmm1
-; X64-SKX-LARGE-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-LARGE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-SKX-LARGE-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-LARGE-NEXT: vpgatherqd 72(,%zmm1), %ymm0 {%k1}
; X64-SKX-LARGE-NEXT: retq
;
@@ -432,8 +432,8 @@ define <8 x i32> @test9(ptr %base, <8 x i64> %ind1, <8 x i32>%ind5) {
; X86-SKX-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm0
; X86-SKX-NEXT: vpaddd {{[0-9]+}}(%esp){1to8}, %ymm0, %ymm0
; X86-SKX-NEXT: vpaddd %ymm1, %ymm0, %ymm1
-; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpgatherdd 68(,%ymm1), %ymm0 {%k1}
; X86-SKX-NEXT: retl
entry:
@@ -489,8 +489,8 @@ define <8 x i32> @test10(ptr %base, <8 x i64> %i1, <8 x i32>%ind5) {
; X64-SKX-SMALL-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero
; X64-SKX-SMALL-NEXT: vpmuldq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm1, %zmm1
; X64-SKX-SMALL-NEXT: vpaddq %zmm1, %zmm0, %zmm1
-; X64-SKX-SMALL-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-SMALL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-SKX-SMALL-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-SMALL-NEXT: vpgatherqd 72(,%zmm1), %ymm0 {%k1}
; X64-SKX-SMALL-NEXT: retq
;
@@ -504,8 +504,8 @@ define <8 x i32> @test10(ptr %base, <8 x i64> %i1, <8 x i32>%ind5) {
; X64-SKX-LARGE-NEXT: vpmullq (%rax){1to8}, %zmm0, %zmm0
; X64-SKX-LARGE-NEXT: vpaddq %zmm0, %zmm2, %zmm0
; X64-SKX-LARGE-NEXT: vpaddq %zmm1, %zmm0, %zmm1
-; X64-SKX-LARGE-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-LARGE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-SKX-LARGE-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-LARGE-NEXT: vpgatherqd 72(,%zmm1), %ymm0 {%k1}
; X64-SKX-LARGE-NEXT: retq
;
@@ -516,8 +516,8 @@ define <8 x i32> @test10(ptr %base, <8 x i64> %i1, <8 x i32>%ind5) {
; X86-SKX-NEXT: vpmulld {{\.?LCPI[0-9]+_[0-9]+}}{1to8}, %ymm0, %ymm0
; X86-SKX-NEXT: vpaddd {{[0-9]+}}(%esp){1to8}, %ymm0, %ymm0
; X86-SKX-NEXT: vpaddd %ymm1, %ymm0, %ymm1
-; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpgatherdd 68(,%ymm1), %ymm0 {%k1}
; X86-SKX-NEXT: retl
entry:
@@ -564,8 +564,8 @@ define <16 x float> @test11(ptr %base, i32 %ind) {
define <16 x float> @test12(ptr %base, <16 x i32> %ind) {
; X64-LABEL: test12:
; X64: # %bb.0:
-; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}
; X64-NEXT: vmovaps %zmm1, %zmm0
; X64-NEXT: retq
@@ -573,8 +573,8 @@ define <16 x float> @test12(ptr %base, <16 x i32> %ind) {
; X86-LABEL: test12:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vgatherdps (%eax,%zmm0,4), %zmm1 {%k1}
; X86-NEXT: vmovaps %zmm1, %zmm0
; X86-NEXT: retl
@@ -589,8 +589,8 @@ define <16 x float> @test12(ptr %base, <16 x i32> %ind) {
define <16 x float> @test13(ptr %base, <16 x i32> %ind) {
; X64-LABEL: test13:
; X64: # %bb.0:
-; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vgatherdps (%rdi,%zmm0,4), %zmm1 {%k1}
; X64-NEXT: vmovaps %zmm1, %zmm0
; X64-NEXT: retq
@@ -598,8 +598,8 @@ define <16 x float> @test13(ptr %base, <16 x i32> %ind) {
; X86-LABEL: test13:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vgatherdps (%eax,%zmm0,4), %zmm1 {%k1}
; X86-NEXT: vmovaps %zmm1, %zmm0
; X86-NEXT: retl
@@ -616,8 +616,8 @@ define <16 x float> @test14(ptr %base, i32 %ind, <16 x ptr> %vec) {
; X64: # %bb.0:
; X64-NEXT: vmovq %xmm0, %rax
; X64-NEXT: vpbroadcastd %esi, %zmm1
-; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vgatherdps (%rax,%zmm1,4), %zmm0 {%k1}
; X64-NEXT: retq
;
@@ -625,8 +625,8 @@ define <16 x float> @test14(ptr %base, i32 %ind, <16 x ptr> %vec) {
; X86: # %bb.0:
; X86-NEXT: vmovd %xmm0, %eax
; X86-NEXT: vbroadcastss {{[0-9]+}}(%esp), %zmm1
-; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vgatherdps (%eax,%zmm1,4), %zmm0 {%k1}
; X86-NEXT: retl
%broadcast.splatinsert = insertelement <16 x ptr> %vec, ptr %base, i32 1
@@ -2424,16 +2424,16 @@ define <16 x ptr> @test31(<16 x ptr> %ptrs) {
;
; X86-LABEL: test31:
; X86: # %bb.0:
-; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vpgatherdd (,%zmm0), %zmm1 {%k1}
; X86-NEXT: vmovdqa64 %zmm1, %zmm0
; X86-NEXT: retl
;
; X64-SKX-LABEL: test31:
; X64-SKX: # %bb.0:
-; X64-SKX-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; X64-SKX-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-NEXT: kxnorb %k0, %k0, %k2
; X64-SKX-NEXT: vpxor %xmm3, %xmm3, %xmm3
; X64-SKX-NEXT: vpgatherqq (,%zmm0), %zmm3 {%k2}
@@ -3220,8 +3220,8 @@ define <8 x i32> @test_global_array(<8 x i64> %indxs) {
;
; X64-SKX-SMALL-LABEL: test_global_array:
; X64-SKX-SMALL: # %bb.0:
-; X64-SKX-SMALL-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-SMALL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-SKX-SMALL-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-SMALL-NEXT: vpgatherqd glob_array(,%zmm0,4), %ymm1 {%k1}
; X64-SKX-SMALL-NEXT: vmovdqa %ymm1, %ymm0
; X64-SKX-SMALL-NEXT: retq
@@ -3229,16 +3229,16 @@ define <8 x i32> @test_global_array(<8 x i64> %indxs) {
; X64-SKX-LARGE-LABEL: test_global_array:
; X64-SKX-LARGE: # %bb.0:
; X64-SKX-LARGE-NEXT: movabsq $glob_array, %rax
-; X64-SKX-LARGE-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-LARGE-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-SKX-LARGE-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-LARGE-NEXT: vpgatherqd (%rax,%zmm0,4), %ymm1 {%k1}
; X64-SKX-LARGE-NEXT: vmovdqa %ymm1, %ymm0
; X64-SKX-LARGE-NEXT: retq
;
; X86-SKX-LABEL: test_global_array:
; X86-SKX: # %bb.0:
-; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpgatherqd glob_array(,%zmm0,4), %ymm1 {%k1}
; X86-SKX-NEXT: vmovdqa %ymm1, %ymm0
; X86-SKX-NEXT: retl
@@ -3266,8 +3266,8 @@ define <8 x i32> @test_global_array_zeroinitializer_index(<8 x i64> %indxs) {
;
; X64-SKX-SMALL-LABEL: test_global_array_zeroinitializer_index:
; X64-SKX-SMALL: # %bb.0:
-; X64-SKX-SMALL-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-SMALL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-SKX-SMALL-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-SMALL-NEXT: vpgatherqd glob_array(,%zmm0,4), %ymm1 {%k1}
; X64-SKX-SMALL-NEXT: vmovdqa %ymm1, %ymm0
; X64-SKX-SMALL-NEXT: retq
@@ -3275,16 +3275,16 @@ define <8 x i32> @test_global_array_zeroinitializer_index(<8 x i64> %indxs) {
; X64-SKX-LARGE-LABEL: test_global_array_zeroinitializer_index:
; X64-SKX-LARGE: # %bb.0:
; X64-SKX-LARGE-NEXT: movabsq $glob_array, %rax
-; X64-SKX-LARGE-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-LARGE-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-SKX-LARGE-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-LARGE-NEXT: vpgatherqd (%rax,%zmm0,4), %ymm1 {%k1}
; X64-SKX-LARGE-NEXT: vmovdqa %ymm1, %ymm0
; X64-SKX-LARGE-NEXT: retq
;
; X86-SKX-LABEL: test_global_array_zeroinitializer_index:
; X86-SKX: # %bb.0:
-; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpgatherqd glob_array(,%zmm0,4), %ymm1 {%k1}
; X86-SKX-NEXT: vmovdqa %ymm1, %ymm0
; X86-SKX-NEXT: retl
@@ -3472,8 +3472,8 @@ define <16 x float> @sext_i8_index(ptr %base, <16 x i8> %ind) {
; X64-LABEL: sext_i8_index:
; X64: # %bb.0:
; X64-NEXT: vpmovsxbd %xmm0, %zmm1
-; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1}
; X64-NEXT: retq
;
@@ -3481,8 +3481,8 @@ define <16 x float> @sext_i8_index(ptr %base, <16 x i8> %ind) {
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: vpmovsxbd %xmm0, %zmm1
-; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vgatherdps (%eax,%zmm1,4), %zmm0 {%k1}
; X86-NEXT: retl
%sext_ind = sext <16 x i8> %ind to <16 x i64>
@@ -3518,8 +3518,8 @@ define <8 x float> @sext_v8i8_index(ptr %base, <8 x i8> %ind) {
; X64-SKX-LABEL: sext_v8i8_index:
; X64-SKX: # %bb.0:
; X64-SKX-NEXT: vpmovsxbd %xmm0, %ymm1
-; X64-SKX-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-SKX-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-NEXT: vgatherdps (%rdi,%ymm1,4), %ymm0 {%k1}
; X64-SKX-NEXT: retq
;
@@ -3527,8 +3527,8 @@ define <8 x float> @sext_v8i8_index(ptr %base, <8 x i8> %ind) {
; X86-SKX: # %bb.0:
; X86-SKX-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SKX-NEXT: vpmovsxbd %xmm0, %ymm1
-; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vgatherdps (%eax,%ymm1,4), %ymm0 {%k1}
; X86-SKX-NEXT: retl
%sext_ind = sext <8 x i8> %ind to <8 x i64>
@@ -3544,8 +3544,8 @@ define <16 x float> @zext_i8_index(ptr %base, <16 x i8> %ind) {
; X64-LABEL: zext_i8_index:
; X64: # %bb.0:
; X64-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1}
; X64-NEXT: retq
;
@@ -3553,8 +3553,8 @@ define <16 x float> @zext_i8_index(ptr %base, <16 x i8> %ind) {
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vgatherdps (%eax,%zmm1,4), %zmm0 {%k1}
; X86-NEXT: retl
%zext_ind = zext <16 x i8> %ind to <16 x i64>
@@ -3590,8 +3590,8 @@ define <8 x float> @zext_v8i8_index(ptr %base, <8 x i8> %ind) {
; X64-SKX-LABEL: zext_v8i8_index:
; X64-SKX: # %bb.0:
; X64-SKX-NEXT: vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; X64-SKX-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-SKX-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-NEXT: vgatherdps (%rdi,%ymm1,4), %ymm0 {%k1}
; X64-SKX-NEXT: retq
;
@@ -3599,8 +3599,8 @@ define <8 x float> @zext_v8i8_index(ptr %base, <8 x i8> %ind) {
; X86-SKX: # %bb.0:
; X86-SKX-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SKX-NEXT: vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
-; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vgatherdps (%eax,%ymm1,4), %ymm0 {%k1}
; X86-SKX-NEXT: retl
%zext_ind = zext <8 x i8> %ind to <8 x i64>
@@ -3726,8 +3726,8 @@ define <16 x float> @zext_index(ptr %base, <16 x i32> %ind) {
; X64-KNL-LABEL: zext_index:
; X64-KNL: # %bb.0:
; X64-KNL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm1
-; X64-KNL-NEXT: kxnorw %k0, %k0, %k1
; X64-KNL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-KNL-NEXT: kxnorw %k0, %k0, %k1
; X64-KNL-NEXT: vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1}
; X64-KNL-NEXT: retq
;
@@ -3735,16 +3735,16 @@ define <16 x float> @zext_index(ptr %base, <16 x i32> %ind) {
; X86-KNL: # %bb.0:
; X86-KNL-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-KNL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm1
-; X86-KNL-NEXT: kxnorw %k0, %k0, %k1
; X86-KNL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86-KNL-NEXT: kxnorw %k0, %k0, %k1
; X86-KNL-NEXT: vgatherdps (%eax,%zmm1,4), %zmm0 {%k1}
; X86-KNL-NEXT: retl
;
; X64-SKX-SMALL-LABEL: zext_index:
; X64-SKX-SMALL: # %bb.0:
; X64-SKX-SMALL-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm1
-; X64-SKX-SMALL-NEXT: kxnorw %k0, %k0, %k1
; X64-SKX-SMALL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X64-SKX-SMALL-NEXT: kxnorw %k0, %k0, %k1
; X64-SKX-SMALL-NEXT: vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1}
; X64-SKX-SMALL-NEXT: retq
;
@@ -3752,8 +3752,8 @@ define <16 x float> @zext_index(ptr %base, <16 x i32> %ind) {
; X64-SKX-LARGE: # %bb.0:
; X64-SKX-LARGE-NEXT: movabsq ${{\.?LCPI[0-9]+_[0-9]+}}, %rax
; X64-SKX-LARGE-NEXT: vandps (%rax){1to16}, %zmm0, %zmm1
-; X64-SKX-LARGE-NEXT: kxnorw %k0, %k0, %k1
; X64-SKX-LARGE-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X64-SKX-LARGE-NEXT: kxnorw %k0, %k0, %k1
; X64-SKX-LARGE-NEXT: vgatherdps (%rdi,%zmm1,4), %zmm0 {%k1}
; X64-SKX-LARGE-NEXT: retq
;
@@ -3761,8 +3761,8 @@ define <16 x float> @zext_index(ptr %base, <16 x i32> %ind) {
; X86-SKX: # %bb.0:
; X86-SKX-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-SKX-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}{1to16}, %zmm0, %zmm1
-; X86-SKX-NEXT: kxnorw %k0, %k0, %k1
; X86-SKX-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86-SKX-NEXT: kxnorw %k0, %k0, %k1
; X86-SKX-NEXT: vgatherdps (%eax,%zmm1,4), %zmm0 {%k1}
; X86-SKX-NEXT: retl
%ind_masked = and <16 x i32> %ind, <i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15, i32 15>
@@ -4116,8 +4116,8 @@ define <16 x i32> @gather_16i64_constant_indices(ptr %ptr, <16 x i1> %mask) {
; X64-KNL-NEXT: vpmovsxbd %xmm0, %zmm0
; X64-KNL-NEXT: vpslld $31, %zmm0, %zmm0
; X64-KNL-NEXT: vptestmd %zmm0, %zmm0, %k1
-; X64-KNL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
; X64-KNL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-KNL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
; X64-KNL-NEXT: vpgatherdd (%rdi,%zmm1,4), %zmm0 {%k1}
; X64-KNL-NEXT: retq
;
@@ -4127,8 +4127,8 @@ define <16 x i32> @gather_16i64_constant_indices(ptr %ptr, <16 x i1> %mask) {
; X86-KNL-NEXT: vpslld $31, %zmm0, %zmm0
; X86-KNL-NEXT: vptestmd %zmm0, %zmm0, %k1
; X86-KNL-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-KNL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
; X86-KNL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86-KNL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
; X86-KNL-NEXT: vpgatherdd (%eax,%zmm1,4), %zmm0 {%k1}
; X86-KNL-NEXT: retl
;
@@ -4137,8 +4137,8 @@ define <16 x i32> @gather_16i64_constant_indices(ptr %ptr, <16 x i1> %mask) {
; X64-SKX-SMALL-NEXT: vpmovsxbd %xmm0, %zmm0
; X64-SKX-SMALL-NEXT: vpslld $31, %zmm0, %zmm0
; X64-SKX-SMALL-NEXT: vpmovd2m %zmm0, %k1
-; X64-SKX-SMALL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
; X64-SKX-SMALL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-SKX-SMALL-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
; X64-SKX-SMALL-NEXT: vpgatherdd (%rdi,%zmm1,4), %zmm0 {%k1}
; X64-SKX-SMALL-NEXT: retq
;
@@ -4159,8 +4159,8 @@ define <16 x i32> @gather_16i64_constant_indices(ptr %ptr, <16 x i1> %mask) {
; X86-SKX-NEXT: vpslld $31, %zmm0, %zmm0
; X86-SKX-NEXT: vpmovd2m %zmm0, %k1
; X86-SKX-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SKX-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
; X86-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86-SKX-NEXT: vmovdqa64 {{.*#+}} zmm1 = [0,4294967294,1,4294967288,10,20,50,65536,16777215,2147483647,100,4294965296,2147483648,76897723,7,4227069609]
; X86-SKX-NEXT: vpgatherdd (%eax,%zmm1,4), %zmm0 {%k1}
; X86-SKX-NEXT: retl
%gep = getelementptr i32, ptr %ptr, <16 x i64> <i64 0, i64 -2, i64 1, i64 -8, i64 10, i64 20, i64 50, i64 65536, i64 16777215, i64 2147483647, i64 100, i64 -2000, i64 -2147483648, i64 76897723, i64 7, i64 -67897687>
@@ -4734,8 +4734,8 @@ declare void @llvm.masked.scatter.v8f32.v8p0(<8 x float>, <8 x ptr>, i32 immarg,
define <16 x i32> @pr163023_sext(ptr %a0, <16 x i32> %a1) {
; X64-LABEL: pr163023_sext:
; X64: # %bb.0:
-; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-NEXT: kxnorw %k0, %k0, %k1
; X64-NEXT: vpgatherdd (%rdi,%zmm0), %zmm1 {%k1}
; X64-NEXT: vmovdqa64 %zmm1, %zmm0
; X64-NEXT: retq
@@ -4743,8 +4743,8 @@ define <16 x i32> @pr163023_sext(ptr %a0, <16 x i32> %a1) {
; X86-LABEL: pr163023_sext:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vpgatherdd (%eax,%zmm0), %zmm1 {%k1}
; X86-NEXT: vmovdqa64 %zmm1, %zmm0
; X86-NEXT: retl
@@ -4776,8 +4776,8 @@ define <16 x i32> @pr163023_zext(ptr %a0, <16 x i32> %a1) {
; X86-LABEL: pr163023_zext:
; X86: # %bb.0:
; X86-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-NEXT: kxnorw %k0, %k0, %k1
; X86-NEXT: vpgatherdd (%eax,%zmm0), %zmm1 {%k1}
; X86-NEXT: vmovdqa64 %zmm1, %zmm0
; X86-NEXT: retl
@@ -4787,8 +4787,8 @@ define <16 x i32> @pr163023_zext(ptr %a0, <16 x i32> %a1) {
; X64-SKX-NEXT: vpmovzxdq {{.*#+}} zmm1 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
; X64-SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; X64-SKX-NEXT: vpmovzxdq {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero
-; X64-SKX-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; X64-SKX-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-NEXT: kxnorb %k0, %k0, %k2
; X64-SKX-NEXT: vpxor %xmm3, %xmm3, %xmm3
; X64-SKX-NEXT: vpgatherqd (%rdi,%zmm0), %ymm3 {%k2}
@@ -4832,16 +4832,16 @@ define <8 x i64> @pr45906(<8 x ptr> %ptr) {
;
; X64-SKX-LABEL: pr45906:
; X64-SKX: # %bb.0: # %bb
-; X64-SKX-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-SKX-NEXT: kxnorb %k0, %k0, %k1
; X64-SKX-NEXT: vpgatherqq 8(,%zmm0), %zmm1 {%k1}
; X64-SKX-NEXT: vmovdqa64 %zmm1, %zmm0
; X64-SKX-NEXT: retq
;
; X86-SKX-LABEL: pr45906:
; X86-SKX: # %bb.0: # %bb
-; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-SKX-NEXT: kxnorb %k0, %k0, %k1
; X86-SKX-NEXT: vpgatherdq 4(,%ymm0), %zmm1 {%k1}
; X86-SKX-NEXT: vmovdqa64 %zmm1, %zmm0
; X86-SKX-NEXT: retl
diff --git a/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll b/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll
index 82f32898244a6..dd8a2b8398f5e 100644
--- a/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll
+++ b/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll
@@ -659,8 +659,8 @@ define <17 x float> @test_mgather_v17f32(ptr %base, <17 x i32> %index)
; WIDEN_SKX-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; WIDEN_SKX-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; WIDEN_SKX-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_SKX-NEXT: kxnorw %k0, %k0, %k1
; WIDEN_SKX-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; WIDEN_SKX-NEXT: kxnorw %k0, %k0, %k1
; WIDEN_SKX-NEXT: vxorps %xmm3, %xmm3, %xmm3
; WIDEN_SKX-NEXT: vgatherdps (%rsi,%zmm0,4), %zmm3 {%k1}
; WIDEN_SKX-NEXT: movw $1, %cx
@@ -694,8 +694,8 @@ define <17 x float> @test_mgather_v17f32(ptr %base, <17 x i32> %index)
; WIDEN_KNL-NEXT: vinserti128 $1, %xmm2, %ymm1, %ymm1
; WIDEN_KNL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; WIDEN_KNL-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
-; WIDEN_KNL-NEXT: kxnorw %k0, %k0, %k1
; WIDEN_KNL-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; WIDEN_KNL-NEXT: kxnorw %k0, %k0, %k1
; WIDEN_KNL-NEXT: vxorps %xmm3, %xmm3, %xmm3
; WIDEN_KNL-NEXT: vgatherdps (%rsi,%zmm0,4), %zmm3 {%k1}
; WIDEN_KNL-NEXT: movw $1, %cx
@@ -728,18 +728,18 @@ define <17 x float> @test_mgather_v17f32(ptr %base, <17 x i32> %index)
; WIDEN_AVX2-NEXT: vpinsrd $2, {{[0-9]+}}(%rsp), %xmm1, %xmm1
; WIDEN_AVX2-NEXT: vpinsrd $3, {{[0-9]+}}(%rsp), %xmm1, %xmm1
; WIDEN_AVX2-NEXT: vinserti128 $1, %xmm1, %ymm3, %ymm1
-; WIDEN_AVX2-NEXT: vpcmpeqd %ymm3, %ymm3, %ymm3
-; WIDEN_AVX2-NEXT: vxorps %xmm4, %xmm4, %xmm4
-; WIDEN_AVX2-NEXT: vpcmpeqd %ymm5, %ymm5, %ymm5
-; WIDEN_AVX2-NEXT: vxorps %xmm6, %xmm6, %xmm6
-; WIDEN_AVX2-NEXT: vgatherdps %ymm5, (%rsi,%ymm1,4), %ymm6
+; WIDEN_AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; WIDEN_AVX2-NEXT: vpcmpeqd %ymm4, %ymm4, %ymm4
+; WIDEN_AVX2-NEXT: vxorps %xmm5, %xmm5, %xmm5
+; WIDEN_AVX2-NEXT: vpcmpeqd %ymm6, %ymm6, %ymm6
+; WIDEN_AVX2-NEXT: vgatherdps %ymm6, (%rsi,%ymm1,4), %ymm5
; WIDEN_AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; WIDEN_AVX2-NEXT: vgatherdps %ymm3, (%rsi,%ymm0,4), %ymm1
+; WIDEN_AVX2-NEXT: vgatherdps %ymm4, (%rsi,%ymm0,4), %ymm1
; WIDEN_AVX2-NEXT: vmovss {{.*#+}} xmm0 = [4294967295,0,0,0]
-; WIDEN_AVX2-NEXT: vgatherdps %ymm0, (%rsi,%ymm2,4), %ymm4
-; WIDEN_AVX2-NEXT: vmovss %xmm4, 64(%rdi)
+; WIDEN_AVX2-NEXT: vgatherdps %ymm0, (%rsi,%ymm2,4), %ymm3
+; WIDEN_AVX2-NEXT: vmovss %xmm3, 64(%rdi)
; WIDEN_AVX2-NEXT: vmovaps %ymm1, 32(%rdi)
-; WIDEN_AVX2-NEXT: vmovaps %ymm6, (%rdi)
+; WIDEN_AVX2-NEXT: vmovaps %ymm5, (%rdi)
; WIDEN_AVX2-NEXT: vzeroupper
; WIDEN_AVX2-NEXT: retq
{
diff --git a/llvm/test/CodeGen/X86/masked_loadstore_split.ll b/llvm/test/CodeGen/X86/masked_loadstore_split.ll
index 0e689a597a72b..c97823d1926b6 100644
--- a/llvm/test/CodeGen/X86/masked_loadstore_split.ll
+++ b/llvm/test/CodeGen/X86/masked_loadstore_split.ll
@@ -7,10 +7,11 @@ define void @split_masked_store(ptr %0) {
; CHECK-NEXT: liveins: $rdi
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:gr64 = COPY $rdi
- ; CHECK-NEXT: [[AVX_SET0_:%[0-9]+]]:vr256 = AVX_SET0
+ ; CHECK-NEXT: [[V_SET0_:%[0-9]+]]:vr128 = V_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:vr256 = SUBREG_TO_REG killed [[V_SET0_]], %subreg.sub_xmm
; CHECK-NEXT: [[VMOVAPSYrm:%[0-9]+]]:vr256 = VMOVAPSYrm $rip, 1, $noreg, %const.0, $noreg :: (load (s256) from constant-pool)
- ; CHECK-NEXT: VMASKMOVPDYmr [[COPY]], 1, $noreg, 32, $noreg, killed [[VMOVAPSYrm]], [[AVX_SET0_]] :: (store unknown-size into %ir.0 + 32, align 8)
- ; CHECK-NEXT: VMOVUPDYmr [[COPY]], 1, $noreg, 0, $noreg, [[AVX_SET0_]] :: (store (s256) into %ir.0, align 8)
+ ; CHECK-NEXT: VMASKMOVPDYmr [[COPY]], 1, $noreg, 32, $noreg, killed [[VMOVAPSYrm]], [[SUBREG_TO_REG]] :: (store unknown-size into %ir.0 + 32, align 8)
+ ; CHECK-NEXT: VMOVUPDYmr [[COPY]], 1, $noreg, 0, $noreg, [[SUBREG_TO_REG]] :: (store (s256) into %ir.0, align 8)
; CHECK-NEXT: RET 0
entry:
call void @llvm.masked.store.v8f64.p0(<8 x double> zeroinitializer, ptr %0, i32 8, <8 x i1> <i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 false, i1 false>)
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index 4c89fa95fe4b2..1b261b8981c84 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -172,8 +172,8 @@ define dso_local void @psubus_64i8_max_512(ptr %xptr, ptr %yptr, ptr %zptr) "min
define dso_local i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocapture readonly, i32) "min-legal-vector-width"="256" {
; CHECK-SKX-LABEL: _Z9test_charPcS_i_256:
; CHECK-SKX: # %bb.0: # %entry
-; CHECK-SKX-NEXT: movl %edx, %eax
; CHECK-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; CHECK-SKX-NEXT: movl %edx, %eax
; CHECK-SKX-NEXT: xorl %ecx, %ecx
; CHECK-SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-SKX-NEXT: vpxor %xmm2, %xmm2, %xmm2
@@ -207,8 +207,8 @@ define dso_local i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocaptur
;
; CHECK-AVX512-LABEL: _Z9test_charPcS_i_256:
; CHECK-AVX512: # %bb.0: # %entry
-; CHECK-AVX512-NEXT: movl %edx, %eax
; CHECK-AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; CHECK-AVX512-NEXT: movl %edx, %eax
; CHECK-AVX512-NEXT: xorl %ecx, %ecx
; CHECK-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
@@ -242,8 +242,8 @@ define dso_local i32 @_Z9test_charPcS_i_256(ptr nocapture readonly, ptr nocaptur
;
; CHECK-VBMI-LABEL: _Z9test_charPcS_i_256:
; CHECK-VBMI: # %bb.0: # %entry
-; CHECK-VBMI-NEXT: movl %edx, %eax
; CHECK-VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; CHECK-VBMI-NEXT: movl %edx, %eax
; CHECK-VBMI-NEXT: xorl %ecx, %ecx
; CHECK-VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-VBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2
@@ -303,8 +303,8 @@ middle.block:
define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocapture readonly, i32) "min-legal-vector-width"="512" {
; CHECK-SKX-LABEL: _Z9test_charPcS_i_512:
; CHECK-SKX: # %bb.0: # %entry
-; CHECK-SKX-NEXT: movl %edx, %eax
; CHECK-SKX-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; CHECK-SKX-NEXT: movl %edx, %eax
; CHECK-SKX-NEXT: xorl %ecx, %ecx
; CHECK-SKX-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-SKX-NEXT: .p2align 4
@@ -333,8 +333,8 @@ define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocaptur
;
; CHECK-AVX512-LABEL: _Z9test_charPcS_i_512:
; CHECK-AVX512: # %bb.0: # %entry
-; CHECK-AVX512-NEXT: movl %edx, %eax
; CHECK-AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; CHECK-AVX512-NEXT: movl %edx, %eax
; CHECK-AVX512-NEXT: xorl %ecx, %ecx
; CHECK-AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-AVX512-NEXT: .p2align 4
@@ -363,8 +363,8 @@ define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocaptur
;
; CHECK-VBMI-LABEL: _Z9test_charPcS_i_512:
; CHECK-VBMI: # %bb.0: # %entry
-; CHECK-VBMI-NEXT: movl %edx, %eax
; CHECK-VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; CHECK-VBMI-NEXT: movl %edx, %eax
; CHECK-VBMI-NEXT: xorl %ecx, %ecx
; CHECK-VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1
; CHECK-VBMI-NEXT: .p2align 4
@@ -1937,8 +1937,8 @@ define <16 x i8> @var_rotate_v16i8(<16 x i8> %a, <16 x i8> %b) nounwind "min-leg
define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind "min-legal-vector-width"="256" {
; CHECK-LABEL: var_rotate_v32i8:
; CHECK: # %bb.0:
-; CHECK-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; CHECK-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; CHECK-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; CHECK-NEXT: vpsllvw %ymm3, %ymm4, %ymm3
diff --git a/llvm/test/CodeGen/X86/non-foldable-with-the-same-mask.mir b/llvm/test/CodeGen/X86/non-foldable-with-the-same-mask.mir
index 95cb460495512..5dfc894257a3a 100644
--- a/llvm/test/CodeGen/X86/non-foldable-with-the-same-mask.mir
+++ b/llvm/test/CodeGen/X86/non-foldable-with-the-same-mask.mir
@@ -384,10 +384,12 @@ name: test_vinserti32x4_same_mask
tracksRegLiveness: true
registers:
- { id: 0, class: vk4wm }
- - { id: 1, class: vr256x }
+ - { id: 1, class: vr128x }
- { id: 2, class: vr256x }
- { id: 3, class: vr128x }
- - { id: 4, class: vr256 }
+ - { id: 4, class: vr256x }
+ - { id: 5, class: vr128x }
+ - { id: 6, class: vr256 }
liveins:
- { reg: '$rdi' }
- { reg: '$k1' }
@@ -398,18 +400,22 @@ body: |
; CHECK: liveins: $rdi, $k1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vk4wm = COPY $k1
- ; CHECK-NEXT: [[AVX512_256_SET0_:%[0-9]+]]:vr256x = AVX512_256_SET0
- ; CHECK-NEXT: [[AVX512_256_SET0_1:%[0-9]+]]:vr256x = AVX512_256_SET0
+ ; CHECK-NEXT: [[AVX512_128_SET0_:%[0-9]+]]:vr128x = AVX512_128_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:vr256x = SUBREG_TO_REG [[AVX512_128_SET0_]], %subreg.sub_xmm
+ ; CHECK-NEXT: [[AVX512_128_SET0_1:%[0-9]+]]:vr128x = AVX512_128_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:vr256x = SUBREG_TO_REG [[AVX512_128_SET0_1]], %subreg.sub_xmm
; CHECK-NEXT: [[VMOVDQA32Z128rmkz:%[0-9]+]]:vr128x = VMOVDQA32Z128rmkz [[COPY]], $rdi, 1, $noreg, 0, $noreg :: (load (s128))
- ; CHECK-NEXT: [[VINSERTI32X4Z256rrikz:%[0-9]+]]:vr256 = VINSERTI32X4Z256rrikz [[COPY]], [[AVX512_256_SET0_1]], [[VMOVDQA32Z128rmkz]], 1
+ ; CHECK-NEXT: [[VINSERTI32X4Z256rrikz:%[0-9]+]]:vr256 = VINSERTI32X4Z256rrikz [[COPY]], [[SUBREG_TO_REG1]], [[VMOVDQA32Z128rmkz]], 1
; CHECK-NEXT: $ymm0 = COPY [[VINSERTI32X4Z256rrikz]]
; CHECK-NEXT: RET 0, $ymm0
%0:vk4wm = COPY $k1
- %1:vr256x = AVX512_256_SET0
- %2:vr256x = AVX512_256_SET0
- %3:vr128x = VMOVDQA32Z128rmkz %0, $rdi, 1, $noreg, 0, $noreg :: (load (s128))
- %4:vr256 = VINSERTI32X4Z256rrikz %0, %2, %3, 1
- $ymm0 = COPY %4
+ %1:vr128x = AVX512_128_SET0
+ %2:vr256x = SUBREG_TO_REG %1, %subreg.sub_xmm
+ %3:vr128x = AVX512_128_SET0
+ %4:vr256x = SUBREG_TO_REG %3, %subreg.sub_xmm
+ %5:vr128x = VMOVDQA32Z128rmkz %0, $rdi, 1, $noreg, 0, $noreg :: (load (s128))
+ %6:vr256 = VINSERTI32X4Z256rrikz %0, %4, %5, 1
+ $ymm0 = COPY %6
RET 0, $ymm0
...
---
@@ -450,10 +456,12 @@ name: test_vpermd_same_mask
tracksRegLiveness: true
registers:
- { id: 0, class: vk8wm }
- - { id: 1, class: vr256x }
+ - { id: 1, class: vr128x }
- { id: 2, class: vr256x }
- { id: 3, class: vr256x }
- - { id: 4, class: vr256 }
+ - { id: 4, class: vr128x }
+ - { id: 5, class: vr256x }
+ - { id: 6, class: vr256 }
liveins:
- { reg: '$rdi' }
- { reg: '$k1' }
@@ -464,18 +472,22 @@ body: |
; CHECK: liveins: $rdi, $k1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vk8wm = COPY $k1
- ; CHECK-NEXT: [[AVX512_256_SET0_:%[0-9]+]]:vr256x = AVX512_256_SET0
+ ; CHECK-NEXT: [[AVX512_128_SET0_:%[0-9]+]]:vr128x = AVX512_128_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:vr256x = SUBREG_TO_REG [[AVX512_128_SET0_]], %subreg.sub_xmm
; CHECK-NEXT: [[VMOVDQA32Z256rmkz:%[0-9]+]]:vr256x = VMOVDQA32Z256rmkz [[COPY]], $rdi, 1, $noreg, 0, $noreg :: (load (s256))
- ; CHECK-NEXT: [[AVX512_256_SET0_1:%[0-9]+]]:vr256x = AVX512_256_SET0
- ; CHECK-NEXT: [[VPERMDZ256rrk:%[0-9]+]]:vr256 = VPERMDZ256rrk [[AVX512_256_SET0_]], [[COPY]], [[AVX512_256_SET0_1]], [[VMOVDQA32Z256rmkz]]
+ ; CHECK-NEXT: [[AVX512_128_SET0_1:%[0-9]+]]:vr128x = AVX512_128_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:vr256x = SUBREG_TO_REG [[AVX512_128_SET0_1]], %subreg.sub_xmm
+ ; CHECK-NEXT: [[VPERMDZ256rrk:%[0-9]+]]:vr256 = VPERMDZ256rrk [[SUBREG_TO_REG]], [[COPY]], [[SUBREG_TO_REG1]], [[VMOVDQA32Z256rmkz]]
; CHECK-NEXT: $ymm0 = COPY [[VPERMDZ256rrk]]
; CHECK-NEXT: RET 0, $ymm0
%0:vk8wm = COPY $k1
- %1:vr256x = AVX512_256_SET0
- %2:vr256x = VMOVDQA32Z256rmkz %0, $rdi, 1, $noreg, 0, $noreg :: (load (s256))
- %3:vr256x = AVX512_256_SET0
- %4:vr256 = VPERMDZ256rrk %1, %0, %3, %2
- $ymm0 = COPY %4
+ %1:vr128x = AVX512_128_SET0
+ %2:vr256x = SUBREG_TO_REG %1, %subreg.sub_xmm
+ %3:vr256x = VMOVDQA32Z256rmkz %0, $rdi, 1, $noreg, 0, $noreg :: (load (s256))
+ %4:vr128x = AVX512_128_SET0
+ %5:vr256x = SUBREG_TO_REG %4, %subreg.sub_xmm
+ %6:vr256 = VPERMDZ256rrk %2, %0, %5, %3
+ $ymm0 = COPY %6
RET 0, $ymm0
...
---
@@ -543,10 +555,12 @@ name: test_vpermps_same_mask
tracksRegLiveness: true
registers:
- { id: 0, class: vk8wm }
- - { id: 1, class: vr256x }
+ - { id: 1, class: vr128x }
- { id: 2, class: vr256x }
- { id: 3, class: vr256x }
- - { id: 4, class: vr256 }
+ - { id: 4, class: vr128x }
+ - { id: 5, class: vr256x }
+ - { id: 6, class: vr256 }
liveins:
- { reg: '$rdi' }
- { reg: '$k1' }
@@ -557,18 +571,22 @@ body: |
; CHECK: liveins: $rdi, $k1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vk8wm = COPY $k1
- ; CHECK-NEXT: [[AVX512_256_SET0_:%[0-9]+]]:vr256x = AVX512_256_SET0
+ ; CHECK-NEXT: [[AVX512_128_SET0_:%[0-9]+]]:vr128x = AVX512_128_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:vr256x = SUBREG_TO_REG [[AVX512_128_SET0_]], %subreg.sub_xmm
; CHECK-NEXT: [[VMOVAPSZ256rmkz:%[0-9]+]]:vr256x = VMOVAPSZ256rmkz [[COPY]], $rdi, 1, $noreg, 0, $noreg :: (load (s256))
- ; CHECK-NEXT: [[AVX512_256_SET0_1:%[0-9]+]]:vr256x = AVX512_256_SET0
- ; CHECK-NEXT: [[VPERMPSZ256rrkz:%[0-9]+]]:vr256 = VPERMPSZ256rrkz [[COPY]], [[AVX512_256_SET0_1]], [[VMOVAPSZ256rmkz]]
+ ; CHECK-NEXT: [[AVX512_128_SET0_1:%[0-9]+]]:vr128x = AVX512_128_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:vr256x = SUBREG_TO_REG [[AVX512_128_SET0_1]], %subreg.sub_xmm
+ ; CHECK-NEXT: [[VPERMPSZ256rrkz:%[0-9]+]]:vr256 = VPERMPSZ256rrkz [[COPY]], [[SUBREG_TO_REG1]], [[VMOVAPSZ256rmkz]]
; CHECK-NEXT: $ymm0 = COPY [[VPERMPSZ256rrkz]]
; CHECK-NEXT: RET 0, $ymm0
%0:vk8wm = COPY $k1
- %1:vr256x = AVX512_256_SET0
- %2:vr256x = VMOVAPSZ256rmkz %0, $rdi, 1, $noreg, 0, $noreg :: (load (s256))
- %3:vr256x = AVX512_256_SET0
- %4:vr256 = VPERMPSZ256rrkz %0, %3, %2
- $ymm0 = COPY %4
+ %1:vr128x = AVX512_128_SET0
+ %2:vr256x = SUBREG_TO_REG %1, %subreg.sub_xmm
+ %3:vr256x = VMOVAPSZ256rmkz %0, $rdi, 1, $noreg, 0, $noreg :: (load (s256))
+ %4:vr128x = AVX512_128_SET0
+ %5:vr256x = SUBREG_TO_REG %4, %subreg.sub_xmm
+ %6:vr256 = VPERMPSZ256rrkz %0, %5, %3
+ $ymm0 = COPY %6
RET 0, $ymm0
...
---
@@ -723,10 +741,12 @@ name: test_vshuff32x4_same_mask
tracksRegLiveness: true
registers:
- { id: 0, class: vk8wm }
- - { id: 1, class: vr256x }
+ - { id: 1, class: vr128x }
- { id: 2, class: vr256x }
- { id: 3, class: vr256x }
- - { id: 4, class: vr256 }
+ - { id: 4, class: vr128x }
+ - { id: 5, class: vr256x }
+ - { id: 6, class: vr256 }
liveins:
- { reg: '$rdi' }
- { reg: '$k1' }
@@ -737,18 +757,22 @@ body: |
; CHECK: liveins: $rdi, $k1
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vk8wm = COPY $k1
- ; CHECK-NEXT: [[AVX512_256_SET0_:%[0-9]+]]:vr256x = AVX512_256_SET0
+ ; CHECK-NEXT: [[AVX512_128_SET0_:%[0-9]+]]:vr128x = AVX512_128_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:vr256x = SUBREG_TO_REG [[AVX512_128_SET0_]], %subreg.sub_xmm
; CHECK-NEXT: [[VMOVAPSZ256rmkz:%[0-9]+]]:vr256x = VMOVAPSZ256rmkz [[COPY]], $rdi, 1, $noreg, 0, $noreg :: (load (s256))
- ; CHECK-NEXT: [[AVX512_256_SET0_1:%[0-9]+]]:vr256x = AVX512_256_SET0
- ; CHECK-NEXT: [[VSHUFF32X4Z256rrikz:%[0-9]+]]:vr256 = VSHUFF32X4Z256rrikz [[COPY]], [[AVX512_256_SET0_1]], [[VMOVAPSZ256rmkz]], 2
+ ; CHECK-NEXT: [[AVX512_128_SET0_1:%[0-9]+]]:vr128x = AVX512_128_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG1:%[0-9]+]]:vr256x = SUBREG_TO_REG [[AVX512_128_SET0_1]], %subreg.sub_xmm
+ ; CHECK-NEXT: [[VSHUFF32X4Z256rrikz:%[0-9]+]]:vr256 = VSHUFF32X4Z256rrikz [[COPY]], [[SUBREG_TO_REG1]], [[VMOVAPSZ256rmkz]], 2
; CHECK-NEXT: $ymm0 = COPY [[VSHUFF32X4Z256rrikz]]
; CHECK-NEXT: RET 0, $ymm0
%0:vk8wm = COPY $k1
- %1:vr256x = AVX512_256_SET0
- %2:vr256x = VMOVAPSZ256rmkz %0, $rdi, 1, $noreg, 0, $noreg :: (load (s256))
- %3:vr256x = AVX512_256_SET0
- %4:vr256 = VSHUFF32X4Z256rrikz %0, %3, %2, 2
- $ymm0 = COPY %4
+ %1:vr128x = AVX512_128_SET0
+ %2:vr256x = SUBREG_TO_REG %1, %subreg.sub_xmm
+ %3:vr256x = VMOVAPSZ256rmkz %0, $rdi, 1, $noreg, 0, $noreg :: (load (s256))
+ %4:vr128x = AVX512_128_SET0
+ %5:vr256x = SUBREG_TO_REG %4, %subreg.sub_xmm
+ %6:vr256 = VSHUFF32X4Z256rrikz %0, %5, %3, 2
+ $ymm0 = COPY %6
RET 0, $ymm0
...
---
diff --git a/llvm/test/CodeGen/X86/pr174871.ll b/llvm/test/CodeGen/X86/pr174871.ll
index 9d671a9a1b8d2..e2f44e5f4baa5 100644
--- a/llvm/test/CodeGen/X86/pr174871.ll
+++ b/llvm/test/CodeGen/X86/pr174871.ll
@@ -16,8 +16,8 @@ define <16 x i32> @pr174871(<16 x i32> %a, <16 x i1> %__mask) local_unnamed_addr
; CHECK-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm5
; CHECK-NEXT: vpsubd %zmm2, %zmm0, %zmm6
; CHECK-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm7
-; CHECK-NEXT: vpaddd %zmm0, %zmm0, %zmm8
; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpaddd %zmm0, %zmm0, %zmm8
; CHECK-NEXT: vpbroadcastd {{.*#+}} zmm9 = [4,4,4,4,4,4,4,4,4,4,4,4,4,4,4,4]
; CHECK-NEXT: movw $-2, %cx
; CHECK-NEXT: kmovd %ecx, %k1
@@ -69,7 +69,8 @@ define <16 x i32> @pr174871(<16 x i32> %a, <16 x i1> %__mask) local_unnamed_addr
; CHECK-NEXT: vmovdqa64 %zmm1, %zmm0
; CHECK-NEXT: retq
; CHECK-NEXT: .LBB0_1:
-; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vmovdqa64 %zmm1, %zmm0
; CHECK-NEXT: retq
allocas:
%"internal_mask&function_mask7208" = and <16 x i1> %__mask, <i1 false, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true, i1 true>
diff --git a/llvm/test/CodeGen/X86/pr34592.ll b/llvm/test/CodeGen/X86/pr34592.ll
index 7cbdb39ddf860..e0423d595303e 100644
--- a/llvm/test/CodeGen/X86/pr34592.ll
+++ b/llvm/test/CodeGen/X86/pr34592.ll
@@ -25,6 +25,7 @@ define <16 x i64> @pluto(<16 x i64> %arg, <16 x i64> %arg1, <16 x i64> %arg2, <1
; CHECK-O0-NEXT: vmovaps 16(%rbp), %ymm11
; CHECK-O0-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3,4,5],ymm0[6,7]
; CHECK-O0-NEXT: vxorps %xmm3, %xmm3, %xmm3
+; CHECK-O0-NEXT: # kill: def $ymm3 killed $xmm3
; CHECK-O0-NEXT: vpunpcklqdq {{.*#+}} ymm2 = ymm2[0],ymm1[0],ymm2[2],ymm1[2]
; CHECK-O0-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
; CHECK-O0-NEXT: vpermq {{.*#+}} ymm0 = ymm0[3,1,2,1]
diff --git a/llvm/test/CodeGen/X86/pr45067.ll b/llvm/test/CodeGen/X86/pr45067.ll
index f8190aaa308f1..f9e26e4e4a6fe 100644
--- a/llvm/test/CodeGen/X86/pr45067.ll
+++ b/llvm/test/CodeGen/X86/pr45067.ll
@@ -6,11 +6,11 @@
define void @foo(ptr %x, <8 x i1> %y) {
; CHECK-LABEL: foo:
; CHECK: ## %bb.0:
-; CHECK-NEXT: vpcmpeqd %ymm1, %ymm1, %ymm1
+; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; CHECK-NEXT: movq _global at GOTPCREL(%rip), %rax
-; CHECK-NEXT: vpxor %xmm2, %xmm2, %xmm2
; CHECK-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; CHECK-NEXT: vpgatherdd %ymm1, (%rax,%ymm2), %ymm3
+; CHECK-NEXT: vpgatherdd %ymm2, (%rax,%ymm1), %ymm3
; CHECK-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; CHECK-NEXT: vpslld $31, %ymm0, %ymm0
; CHECK-NEXT: vpmaskmovd %ymm3, %ymm0, (%rdi)
diff --git a/llvm/test/CodeGen/X86/pr51615.ll b/llvm/test/CodeGen/X86/pr51615.ll
index b8dd3c196e22d..29baa56f75edd 100644
--- a/llvm/test/CodeGen/X86/pr51615.ll
+++ b/llvm/test/CodeGen/X86/pr51615.ll
@@ -44,10 +44,10 @@ define void @volatile_load_2_elts() {
define void @volatile_load_1_elt() {
; ALL-LABEL: volatile_load_1_elt:
; ALL: # %bb.0:
-; ALL-NEXT: vbroadcastsd g1(%rip), %ymm0
-; ALL-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; ALL-NEXT: vblendps {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; ALL-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; ALL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ALL-NEXT: vbroadcastsd g1(%rip), %ymm1
+; ALL-NEXT: vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
+; ALL-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
; ALL-NEXT: vmovaps %ymm0, (%rax)
; ALL-NEXT: vmovaps %ymm2, (%rax)
; ALL-NEXT: vzeroupper
@@ -63,10 +63,10 @@ define void @volatile_load_1_elt() {
define void @volatile_load_2_elts_bitcast() {
; ALL-LABEL: volatile_load_2_elts_bitcast:
; ALL: # %bb.0:
-; ALL-NEXT: vbroadcastsd g2(%rip), %ymm0
-; ALL-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; ALL-NEXT: vblendps {{.*#+}} ymm2 = ymm1[0,1,2,3],ymm0[4,5],ymm1[6,7]
-; ALL-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
+; ALL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; ALL-NEXT: vbroadcastsd g2(%rip), %ymm1
+; ALL-NEXT: vblendps {{.*#+}} ymm2 = ymm0[0,1,2,3],ymm1[4,5],ymm0[6,7]
+; ALL-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
; ALL-NEXT: vmovaps %ymm0, (%rax)
; ALL-NEXT: vmovaps %ymm2, (%rax)
; ALL-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/pr53842.ll b/llvm/test/CodeGen/X86/pr53842.ll
index 0c81d03ea62b5..aca1c211e4ff2 100644
--- a/llvm/test/CodeGen/X86/pr53842.ll
+++ b/llvm/test/CodeGen/X86/pr53842.ll
@@ -50,12 +50,13 @@ define void @PR53842_sgt() {
; AVX512F: # %bb.0: # %entry
; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: vpmovsxbq (%rax), %zmm1
-; AVX512F-NEXT: vpcmpgtq %zmm1, %zmm0, %k1
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1
+; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512F-NEXT: .p2align 4
; AVX512F-NEXT: .LBB1_1: # %vector.body
; AVX512F-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512F-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
+; AVX512F-NEXT: vpcmpgtq %zmm1, %zmm0, %k1
+; AVX512F-NEXT: vpsubq %zmm2, %zmm3, %zmm3 {%k1}
; AVX512F-NEXT: jmp .LBB1_1
;
; AVX512DQ-LABEL: PR53842_sgt:
@@ -91,24 +92,26 @@ define void @PR53842_slt() {
; AVX512F: # %bb.0: # %entry
; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512F-NEXT: vpmovsxbq (%rax), %zmm1
-; AVX512F-NEXT: vpcmpgtq %zmm0, %zmm1, %k1
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 = -1
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 = -1
+; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512F-NEXT: .p2align 4
; AVX512F-NEXT: .LBB2_1: # %vector.body
; AVX512F-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512F-NEXT: vpsubq %zmm1, %zmm0, %zmm0 {%k1}
+; AVX512F-NEXT: vpcmpgtq %zmm0, %zmm1, %k1
+; AVX512F-NEXT: vpsubq %zmm2, %zmm3, %zmm3 {%k1}
; AVX512F-NEXT: jmp .LBB2_1
;
; AVX512DQ-LABEL: PR53842_slt:
; AVX512DQ: # %bb.0: # %entry
; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512DQ-NEXT: vpmovsxbq (%rax), %zmm1
-; AVX512DQ-NEXT: vpcmpgtq %zmm0, %zmm1, %k0
-; AVX512DQ-NEXT: vpmovm2q %k0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512DQ-NEXT: .p2align 4
; AVX512DQ-NEXT: .LBB2_1: # %vector.body
; AVX512DQ-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512DQ-NEXT: vpsubq %zmm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: vpcmpgtq %zmm0, %zmm1, %k0
+; AVX512DQ-NEXT: vpmovm2q %k0, %zmm3
+; AVX512DQ-NEXT: vpsubq %zmm3, %zmm2, %zmm2
; AVX512DQ-NEXT: jmp .LBB2_1
entry:
br label %vector.body
diff --git a/llvm/test/CodeGen/X86/pr63108.ll b/llvm/test/CodeGen/X86/pr63108.ll
index b5b80515fc6d9..65e013da4535f 100644
--- a/llvm/test/CodeGen/X86/pr63108.ll
+++ b/llvm/test/CodeGen/X86/pr63108.ll
@@ -54,8 +54,8 @@ define i32 @PR63108() {
; AVX1-NEXT: .LBB0_3: # %vector.body
; AVX1-NEXT: # =>This Inner Loop Header: Depth=1
; AVX1-NEXT: vmovaps %ymm0, %ymm1
-; AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX1-NEXT: testb %al, %al
+; AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX1-NEXT: jne .LBB0_3
; AVX1-NEXT: # %bb.4: # %middle.block
; AVX1-NEXT: vxorps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
@@ -90,8 +90,8 @@ define i32 @PR63108() {
; AVX2-NEXT: .LBB0_3: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
; AVX2-NEXT: vmovdqa %ymm0, %ymm1
-; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX2-NEXT: testb %al, %al
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX2-NEXT: jne .LBB0_3
; AVX2-NEXT: # %bb.4: # %middle.block
; AVX2-NEXT: vpxor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm0
diff --git a/llvm/test/CodeGen/X86/prefer-avx256-lzcnt.ll b/llvm/test/CodeGen/X86/prefer-avx256-lzcnt.ll
index 5f13e97487435..2ab6c8d4746ef 100644
--- a/llvm/test/CodeGen/X86/prefer-avx256-lzcnt.ll
+++ b/llvm/test/CodeGen/X86/prefer-avx256-lzcnt.ll
@@ -92,9 +92,9 @@ define <32 x i8> @testv32i8(<32 x i8> %in) {
; AVX256-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX256-NEXT: # ymm1 = mem[0,1,0,1]
; AVX256-NEXT: vpshufb %ymm0, %ymm1, %ymm2
+; AVX256-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX256-NEXT: vpsrlw $4, %ymm0, %ymm0
; AVX256-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
-; AVX256-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX256-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm3
; AVX256-NEXT: vpand %ymm3, %ymm2, %ymm2
; AVX256-NEXT: vpshufb %ymm0, %ymm1, %ymm0
diff --git a/llvm/test/CodeGen/X86/sadd_sat_vec.ll b/llvm/test/CodeGen/X86/sadd_sat_vec.ll
index a4c94ec2fa883..3120cbd8e1767 100644
--- a/llvm/test/CodeGen/X86/sadd_sat_vec.ll
+++ b/llvm/test/CodeGen/X86/sadd_sat_vec.ll
@@ -893,14 +893,14 @@ define <8 x i32> @v8i32(<8 x i32> %x, <8 x i32> %y) nounwind {
;
; AVX512BW-LABEL: v8i32:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-NEXT: vpcmpgtd %ymm1, %ymm2, %k0
-; AVX512BW-NEXT: vpaddd %ymm1, %ymm0, %ymm1
+; AVX512BW-NEXT: vpaddd %ymm1, %ymm0, %ymm2
+; AVX512BW-NEXT: vpcmpgtd %ymm2, %ymm0, %k0
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512BW-NEXT: vpcmpgtd %ymm1, %ymm0, %k1
-; AVX512BW-NEXT: kxorw %k1, %k0, %k1
-; AVX512BW-NEXT: vpsrad $31, %ymm1, %ymm0
-; AVX512BW-NEXT: vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm1 {%k1}
-; AVX512BW-NEXT: vmovdqa %ymm1, %ymm0
+; AVX512BW-NEXT: kxorw %k0, %k1, %k1
+; AVX512BW-NEXT: vpsrad $31, %ymm2, %ymm0
+; AVX512BW-NEXT: vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm2 {%k1}
+; AVX512BW-NEXT: vmovdqa %ymm2, %ymm0
; AVX512BW-NEXT: retq
%z = call <8 x i32> @llvm.sadd.sat.v8i32(<8 x i32> %x, <8 x i32> %y)
ret <8 x i32> %z
@@ -1101,14 +1101,14 @@ define <16 x i32> @v16i32(<16 x i32> %x, <16 x i32> %y) nounwind {
;
; AVX512-LABEL: v16i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpcmpgtd %zmm1, %zmm2, %k0
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm1
+; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm2
+; AVX512-NEXT: vpcmpgtd %zmm2, %zmm0, %k0
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512-NEXT: vpcmpgtd %zmm1, %zmm0, %k1
-; AVX512-NEXT: kxorw %k1, %k0, %k1
-; AVX512-NEXT: vpsrad $31, %zmm1, %zmm0
-; AVX512-NEXT: vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm1 {%k1}
-; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0
+; AVX512-NEXT: kxorw %k0, %k1, %k1
+; AVX512-NEXT: vpsrad $31, %zmm2, %zmm0
+; AVX512-NEXT: vpxord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm2 {%k1}
+; AVX512-NEXT: vmovdqa64 %zmm2, %zmm0
; AVX512-NEXT: retq
%z = call <16 x i32> @llvm.sadd.sat.v16i32(<16 x i32> %x, <16 x i32> %y)
ret <16 x i32> %z
@@ -1442,14 +1442,14 @@ define <4 x i64> @v4i64(<4 x i64> %x, <4 x i64> %y) nounwind {
;
; AVX512BW-LABEL: v4i64:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-NEXT: vpcmpgtq %ymm1, %ymm2, %k0
-; AVX512BW-NEXT: vpaddq %ymm1, %ymm0, %ymm1
+; AVX512BW-NEXT: vpaddq %ymm1, %ymm0, %ymm2
+; AVX512BW-NEXT: vpcmpgtq %ymm2, %ymm0, %k0
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512BW-NEXT: vpcmpgtq %ymm1, %ymm0, %k1
-; AVX512BW-NEXT: kxorw %k1, %k0, %k1
-; AVX512BW-NEXT: vpsraq $63, %ymm1, %ymm0
-; AVX512BW-NEXT: vpxorq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm1 {%k1}
-; AVX512BW-NEXT: vmovdqa %ymm1, %ymm0
+; AVX512BW-NEXT: kxorw %k0, %k1, %k1
+; AVX512BW-NEXT: vpsraq $63, %ymm2, %ymm0
+; AVX512BW-NEXT: vpxorq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %ymm0, %ymm2 {%k1}
+; AVX512BW-NEXT: vmovdqa %ymm2, %ymm0
; AVX512BW-NEXT: retq
%z = call <4 x i64> @llvm.sadd.sat.v4i64(<4 x i64> %x, <4 x i64> %y)
ret <4 x i64> %z
@@ -1796,14 +1796,14 @@ define <8 x i64> @v8i64(<8 x i64> %x, <8 x i64> %y) nounwind {
;
; AVX512-LABEL: v8i64:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpcmpgtq %zmm1, %zmm2, %k0
-; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm1
+; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm2
+; AVX512-NEXT: vpcmpgtq %zmm2, %zmm0, %k0
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512-NEXT: vpcmpgtq %zmm1, %zmm0, %k1
-; AVX512-NEXT: kxorw %k1, %k0, %k1
-; AVX512-NEXT: vpsraq $63, %zmm1, %zmm0
-; AVX512-NEXT: vpxorq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm1 {%k1}
-; AVX512-NEXT: vmovdqa64 %zmm1, %zmm0
+; AVX512-NEXT: kxorw %k0, %k1, %k1
+; AVX512-NEXT: vpsraq $63, %zmm2, %zmm0
+; AVX512-NEXT: vpxorq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %zmm2 {%k1}
+; AVX512-NEXT: vmovdqa64 %zmm2, %zmm0
; AVX512-NEXT: retq
%z = call <8 x i64> @llvm.sadd.sat.v8i64(<8 x i64> %x, <8 x i64> %y)
ret <8 x i64> %z
diff --git a/llvm/test/CodeGen/X86/shift-i256.ll b/llvm/test/CodeGen/X86/shift-i256.ll
index c731dd6378038..4933ef441de6b 100644
--- a/llvm/test/CodeGen/X86/shift-i256.ll
+++ b/llvm/test/CodeGen/X86/shift-i256.ll
@@ -285,12 +285,12 @@ define i256 @lshr_i256(i256 %a0, i256 %a1) nounwind {
;
; AVX2-LABEL: lshr_i256:
; AVX2: # %bb.0:
-; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %r9d, %eax
; AVX2-NEXT: shrb $6, %al
; AVX2-NEXT: movzbl %al, %eax
@@ -314,12 +314,12 @@ define i256 @lshr_i256(i256 %a0, i256 %a1) nounwind {
;
; AVX512F-LABEL: lshr_i256:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movl %r9d, %eax
; AVX512F-NEXT: shrb $6, %al
; AVX512F-NEXT: movzbl %al, %eax
@@ -342,12 +342,12 @@ define i256 @lshr_i256(i256 %a0, i256 %a1) nounwind {
;
; AVX512VL-LABEL: lshr_i256:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %r9d, %eax
; AVX512VL-NEXT: shrb $6, %al
; AVX512VL-NEXT: movzbl %al, %eax
@@ -371,12 +371,12 @@ define i256 @lshr_i256(i256 %a0, i256 %a1) nounwind {
;
; AVX512VBMI-LABEL: lshr_i256:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %r9d, %eax
; AVX512VBMI-NEXT: shrb $6, %al
; AVX512VBMI-NEXT: movzbl %al, %eax
@@ -1921,10 +1921,10 @@ define i256 @shl_1_i256(i256 %a0) nounwind {
; AVX2-LABEL: shl_1_i256:
; AVX2: # %bb.0:
; AVX2-NEXT: movq %rsi, %rcx
-; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovss {{.*#+}} xmm0 = [1,0,0,0]
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $3, %al
; AVX2-NEXT: andb $24, %al
@@ -1978,10 +1978,10 @@ define i256 @shl_1_i256(i256 %a0) nounwind {
; AVX512VL-LABEL: shl_1_i256:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: movq %rsi, %rcx
-; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %ecx, %eax
; AVX512VL-NEXT: shrb $3, %al
; AVX512VL-NEXT: andb $24, %al
@@ -2007,10 +2007,10 @@ define i256 @shl_1_i256(i256 %a0) nounwind {
; AVX512VBMI-LABEL: shl_1_i256:
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: movq %rsi, %rcx
-; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps {{.*#+}} xmm0 = [1,0,0,0]
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %ecx, %eax
; AVX512VBMI-NEXT: shrb $3, %al
; AVX512VBMI-NEXT: andb $24, %al
@@ -2148,10 +2148,10 @@ define i256 @lshr_signbit_i256(i256 %a0) nounwind {
;
; AVX2-LABEL: lshr_signbit_i256:
; AVX2: # %bb.0:
-; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, %rcx
-; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $6, %al
@@ -2199,11 +2199,11 @@ define i256 @lshr_signbit_i256(i256 %a0) nounwind {
;
; AVX512VL-LABEL: lshr_signbit_i256:
; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, %rcx
; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %ecx, %eax
; AVX512VL-NEXT: shrb $6, %al
; AVX512VL-NEXT: movzbl %al, %eax
@@ -2226,11 +2226,11 @@ define i256 @lshr_signbit_i256(i256 %a0) nounwind {
;
; AVX512VBMI-LABEL: lshr_signbit_i256:
; AVX512VBMI: # %bb.0:
+; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, %rcx
; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %ecx, %eax
; AVX512VBMI-NEXT: shrb $6, %al
; AVX512VBMI-NEXT: movzbl %al, %eax
@@ -2804,10 +2804,10 @@ define i256 @lshr_allbits_i256(i256 %a0) nounwind {
;
; AVX2-LABEL: lshr_allbits_i256:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, %rcx
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %ecx, %eax
; AVX2-NEXT: shrb $6, %al
@@ -2855,10 +2855,10 @@ define i256 @lshr_allbits_i256(i256 %a0) nounwind {
;
; AVX512VL-LABEL: lshr_allbits_i256:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, %rcx
-; AVX512VL-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %ecx, %eax
; AVX512VL-NEXT: shrb $6, %al
@@ -2882,10 +2882,10 @@ define i256 @lshr_allbits_i256(i256 %a0) nounwind {
;
; AVX512VBMI-LABEL: lshr_allbits_i256:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, %rcx
-; AVX512VBMI-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %ecx, %eax
; AVX512VBMI-NEXT: shrb $6, %al
@@ -3002,12 +3002,12 @@ define i64 @lshr_extract_i256_i64(i256 %a0, i256 %a1) nounwind {
;
; AVX2-LABEL: lshr_extract_i256_i64:
; AVX2: # %bb.0:
-; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %r8d, %eax
; AVX2-NEXT: shrb $6, %al
; AVX2-NEXT: movzbl %al, %ecx
@@ -3020,12 +3020,12 @@ define i64 @lshr_extract_i256_i64(i256 %a0, i256 %a1) nounwind {
;
; AVX512F-LABEL: lshr_extract_i256_i64:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movl %r8d, %eax
; AVX512F-NEXT: shrb $6, %al
; AVX512F-NEXT: movzbl %al, %ecx
@@ -3037,12 +3037,12 @@ define i64 @lshr_extract_i256_i64(i256 %a0, i256 %a1) nounwind {
;
; AVX512VL-LABEL: lshr_extract_i256_i64:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %r8d, %eax
; AVX512VL-NEXT: shrb $6, %al
; AVX512VL-NEXT: movzbl %al, %ecx
@@ -3055,12 +3055,12 @@ define i64 @lshr_extract_i256_i64(i256 %a0, i256 %a1) nounwind {
;
; AVX512VBMI-LABEL: lshr_extract_i256_i64:
; AVX512VBMI: # %bb.0:
-; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %r8d, %eax
; AVX512VBMI-NEXT: shrb $6, %al
; AVX512VBMI-NEXT: movzbl %al, %ecx
diff --git a/llvm/test/CodeGen/X86/shift-i512.ll b/llvm/test/CodeGen/X86/shift-i512.ll
index ecbab7d022092..4f1adab266822 100644
--- a/llvm/test/CodeGen/X86/shift-i512.ll
+++ b/llvm/test/CodeGen/X86/shift-i512.ll
@@ -170,34 +170,33 @@ define i512 @shl_i512(i512 %a0, i512 %a1) nounwind {
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: pushq %rax
; AVX512VL-NEXT: movl {{[0-9]+}}(%rsp), %eax
-; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
+; AVX512VL-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512VL-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovups %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovdqu %xmm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %eax, %ecx
; AVX512VL-NEXT: andl $63, %ecx
-; AVX512VL-NEXT: vpbroadcastq %rcx, %xmm0
+; AVX512VL-NEXT: vpbroadcastq %rcx, %xmm1
; AVX512VL-NEXT: shrl $3, %eax
; AVX512VL-NEXT: andl $56, %eax
; AVX512VL-NEXT: negl %eax
; AVX512VL-NEXT: cltq
-; AVX512VL-NEXT: vmovdqu64 -64(%rsp,%rax), %zmm1
-; AVX512VL-NEXT: vpsllq %xmm0, %zmm1, %zmm2
-; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT: valignq {{.*#+}} zmm1 = zmm3[7],zmm1[0,1,2,3,4,5,6]
-; AVX512VL-NEXT: vpsrlq $1, %zmm1, %zmm1
-; AVX512VL-NEXT: vpandnq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; AVX512VL-NEXT: vpsrlq %xmm0, %zmm1, %zmm0
+; AVX512VL-NEXT: vmovdqu64 -64(%rsp,%rax), %zmm2
+; AVX512VL-NEXT: vpsllq %xmm1, %zmm2, %zmm3
+; AVX512VL-NEXT: valignq {{.*#+}} zmm0 = zmm0[7],zmm2[0,1,2,3,4,5,6]
+; AVX512VL-NEXT: vpsrlq $1, %zmm0, %zmm0
+; AVX512VL-NEXT: vpandnq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm1
+; AVX512VL-NEXT: vpsrlq %xmm1, %zmm0, %zmm0
; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: vporq %zmm0, %zmm2, %zmm0
+; AVX512VL-NEXT: vporq %zmm0, %zmm3, %zmm0
; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, 32(%rdi)
; AVX512VL-NEXT: vmovdqu %ymm0, (%rdi)
; AVX512VL-NEXT: popq %rcx
@@ -208,31 +207,30 @@ define i512 @shl_i512(i512 %a0, i512 %a1) nounwind {
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: pushq %rax
; AVX512VBMI-NEXT: movq %rdi, %rax
+; AVX512VBMI-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
+; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; AVX512VBMI-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovdqu %xmm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl {{[0-9]+}}(%rsp), %edi
-; AVX512VBMI-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
-; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX512VBMI-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovups %xmm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %edi, %ecx
; AVX512VBMI-NEXT: shrl $3, %edi
; AVX512VBMI-NEXT: andl $56, %edi
; AVX512VBMI-NEXT: negl %edi
; AVX512VBMI-NEXT: movslq %edi, %rdx
-; AVX512VBMI-NEXT: vmovdqu64 -64(%rsp,%rdx), %zmm0
-; AVX512VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VBMI-NEXT: valignq {{.*#+}} zmm1 = zmm1[7],zmm0[0,1,2,3,4,5,6]
+; AVX512VBMI-NEXT: vmovdqu64 -64(%rsp,%rdx), %zmm1
+; AVX512VBMI-NEXT: valignq {{.*#+}} zmm0 = zmm0[7],zmm1[0,1,2,3,4,5,6]
; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm2
-; AVX512VBMI-NEXT: vpshldvq %zmm2, %zmm1, %zmm0
-; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm0, 32(%rax)
-; AVX512VBMI-NEXT: vmovdqu %ymm0, (%rax)
+; AVX512VBMI-NEXT: vpshldvq %zmm2, %zmm0, %zmm1
+; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm1, 32(%rax)
+; AVX512VBMI-NEXT: vmovdqu %ymm1, (%rax)
; AVX512VBMI-NEXT: popq %rcx
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
@@ -302,19 +300,19 @@ define i512 @lshr_i512(i512 %a0, i512 %a1) nounwind {
; AVX2-NEXT: pushq %r15
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
-; AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
-; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movq {{[0-9]+}}(%rsp), %rax
+; AVX2-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %xmm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %eax, %ecx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %eax
@@ -354,30 +352,30 @@ define i512 @lshr_i512(i512 %a0, i512 %a1) nounwind {
; AVX512F-LABEL: lshr_i512:
; AVX512F: # %bb.0:
; AVX512F-NEXT: pushq %rax
-; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vmovdqu64 %zmm1, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512F-NEXT: movq %rax, -{{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vmovups %xmm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movl {{[0-9]+}}(%rsp), %eax
+; AVX512F-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
+; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; AVX512F-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vmovdqu %xmm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vmovdqu64 %zmm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movl %eax, %ecx
; AVX512F-NEXT: andl $63, %ecx
-; AVX512F-NEXT: vmovq %rcx, %xmm0
-; AVX512F-NEXT: vpbroadcastq %xmm0, %xmm0
+; AVX512F-NEXT: vmovq %rcx, %xmm1
+; AVX512F-NEXT: vpbroadcastq %xmm1, %xmm1
; AVX512F-NEXT: shrl $3, %eax
; AVX512F-NEXT: andl $56, %eax
; AVX512F-NEXT: vmovdqu64 -128(%rsp,%rax), %zmm2
-; AVX512F-NEXT: vpsrlq %xmm0, %zmm2, %zmm3
-; AVX512F-NEXT: valignq {{.*#+}} zmm1 = zmm2[1,2,3,4,5,6,7],zmm1[0]
-; AVX512F-NEXT: vpaddq %zmm1, %zmm1, %zmm1
-; AVX512F-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512F-NEXT: vpsllq %xmm0, %zmm1, %zmm0
+; AVX512F-NEXT: vpsrlq %xmm1, %zmm2, %zmm3
+; AVX512F-NEXT: valignq {{.*#+}} zmm0 = zmm2[1,2,3,4,5,6,7],zmm0[0]
+; AVX512F-NEXT: vpaddq %zmm0, %zmm0, %zmm0
+; AVX512F-NEXT: vpandn {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX512F-NEXT: vpsllq %xmm1, %zmm0, %zmm0
; AVX512F-NEXT: movq %rdi, %rax
; AVX512F-NEXT: vporq %zmm3, %zmm0, %zmm0
; AVX512F-NEXT: vmovdqu64 %zmm0, (%rdi)
@@ -387,33 +385,32 @@ define i512 @lshr_i512(i512 %a0, i512 %a1) nounwind {
; AVX512VL-LABEL: lshr_i512:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: pushq %rax
-; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
; AVX512VL-NEXT: movl {{[0-9]+}}(%rsp), %eax
-; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512VL-NEXT: movq %r10, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovups %xmm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovdqu %xmm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %eax, %ecx
; AVX512VL-NEXT: andl $63, %ecx
-; AVX512VL-NEXT: vpbroadcastq %rcx, %xmm0
+; AVX512VL-NEXT: vpbroadcastq %rcx, %xmm1
; AVX512VL-NEXT: shrl $3, %eax
; AVX512VL-NEXT: andl $56, %eax
-; AVX512VL-NEXT: vmovdqu64 -128(%rsp,%rax), %zmm1
-; AVX512VL-NEXT: vpsrlq %xmm0, %zmm1, %zmm2
-; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT: valignq {{.*#+}} zmm1 = zmm1[1,2,3,4,5,6,7],zmm3[0]
-; AVX512VL-NEXT: vpaddq %zmm1, %zmm1, %zmm1
-; AVX512VL-NEXT: vpandnq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; AVX512VL-NEXT: vpsllq %xmm0, %zmm1, %zmm0
+; AVX512VL-NEXT: vmovdqu64 -128(%rsp,%rax), %zmm2
+; AVX512VL-NEXT: vpsrlq %xmm1, %zmm2, %zmm3
+; AVX512VL-NEXT: valignq {{.*#+}} zmm0 = zmm2[1,2,3,4,5,6,7],zmm0[0]
+; AVX512VL-NEXT: vpaddq %zmm0, %zmm0, %zmm0
+; AVX512VL-NEXT: vpandnq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm1, %xmm1
+; AVX512VL-NEXT: vpsllq %xmm1, %zmm0, %zmm0
; AVX512VL-NEXT: movq %rdi, %rax
-; AVX512VL-NEXT: vporq %zmm2, %zmm0, %zmm0
+; AVX512VL-NEXT: vporq %zmm3, %zmm0, %zmm0
; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, 32(%rdi)
; AVX512VL-NEXT: vmovdqu %ymm0, (%rdi)
; AVX512VL-NEXT: popq %rcx
@@ -424,29 +421,28 @@ define i512 @lshr_i512(i512 %a0, i512 %a1) nounwind {
; AVX512VBMI: # %bb.0:
; AVX512VBMI-NEXT: pushq %rax
; AVX512VBMI-NEXT: movq %rdi, %rax
-; AVX512VBMI-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
-; AVX512VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VBMI-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovdqa {{[0-9]+}}(%rsp), %xmm0
; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %rdi
-; AVX512VBMI-NEXT: vmovdqu %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovups %xmm0, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl {{[0-9]+}}(%rsp), %edi
+; AVX512VBMI-NEXT: vmovdqu %xmm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: movl %edi, %ecx
-; AVX512VBMI-NEXT: shrl $3, %edi
-; AVX512VBMI-NEXT: andl $56, %edi
-; AVX512VBMI-NEXT: vmovdqu64 -128(%rsp,%rdi), %zmm0
-; AVX512VBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VBMI-NEXT: valignq {{.*#+}} zmm1 = zmm0[1,2,3,4,5,6,7],zmm1[0]
-; AVX512VBMI-NEXT: vpbroadcastq %rcx, %zmm2
-; AVX512VBMI-NEXT: vpshrdvq %zmm2, %zmm1, %zmm0
-; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm0, 32(%rax)
-; AVX512VBMI-NEXT: vmovdqu %ymm0, (%rax)
+; AVX512VBMI-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: movl {{[0-9]+}}(%rsp), %ecx
+; AVX512VBMI-NEXT: movl %ecx, %edx
+; AVX512VBMI-NEXT: shrl $3, %ecx
+; AVX512VBMI-NEXT: andl $56, %ecx
+; AVX512VBMI-NEXT: vmovdqu64 -128(%rsp,%rcx), %zmm1
+; AVX512VBMI-NEXT: valignq {{.*#+}} zmm0 = zmm1[1,2,3,4,5,6,7],zmm0[0]
+; AVX512VBMI-NEXT: vpbroadcastq %rdx, %zmm2
+; AVX512VBMI-NEXT: vpshrdvq %zmm2, %zmm0, %zmm1
+; AVX512VBMI-NEXT: vextracti64x4 $1, %zmm1, 32(%rax)
+; AVX512VBMI-NEXT: vmovdqu %ymm1, (%rax)
; AVX512VBMI-NEXT: popq %rcx
; AVX512VBMI-NEXT: vzeroupper
; AVX512VBMI-NEXT: retq
@@ -1715,10 +1711,10 @@ define i512 @shl_1_i512(i512 %a0) nounwind {
; AVX2-NEXT: pushq %rax
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovss {{.*#+}} xmm1 = [1,0,0,0]
+; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovss {{.*#+}} xmm0 = [1,0,0,0]
-; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %esi, %ecx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: shrl $3, %esi
@@ -1861,11 +1857,11 @@ define i512 @lshr_signbit_i512(i512 %a0) nounwind {
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: pushq %rax
+; AVX2-NEXT: vmovaps {{.*#+}} ymm0 = [0,0,0,9223372036854775808]
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovaps {{.*#+}} ymm1 = [0,0,0,9223372036854775808]
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %esi, %ecx
; AVX2-NEXT: andl $63, %ecx
@@ -2345,10 +2341,10 @@ define i512 @lshr_allbits_i512(i512 %a0) nounwind {
; AVX2-NEXT: pushq %r14
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: pushq %rax
-; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovdqu %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %esi, %ecx
@@ -2474,18 +2470,18 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
; AVX2-LABEL: lshr_extract_i512_i64:
; AVX2: # %bb.0:
; AVX2-NEXT: pushq %rax
-; AVX2-NEXT: movl {{[0-9]+}}(%rsp), %r10d
; AVX2-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
-; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: movl {{[0-9]+}}(%rsp), %r10d
; AVX2-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rcx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: movl %r10d, %ecx
; AVX2-NEXT: shrl $3, %ecx
; AVX2-NEXT: andl $56, %ecx
@@ -2502,8 +2498,6 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
; AVX512F-NEXT: pushq %rax
; AVX512F-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
-; AVX512F-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vmovups %zmm1, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
@@ -2511,6 +2505,8 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
; AVX512F-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vmovups %zmm0, -{{[0-9]+}}(%rsp)
; AVX512F-NEXT: movl %r10d, %ecx
; AVX512F-NEXT: shrl $3, %ecx
; AVX512F-NEXT: andl $56, %ecx
@@ -2526,9 +2522,6 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
; AVX512VL-NEXT: pushq %rax
; AVX512VL-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
-; AVX512VL-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
@@ -2536,6 +2529,9 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
; AVX512VL-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VL-NEXT: movl %r10d, %ecx
; AVX512VL-NEXT: shrl $3, %ecx
; AVX512VL-NEXT: andl $56, %ecx
@@ -2552,9 +2548,6 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
; AVX512VBMI-NEXT: pushq %rax
; AVX512VBMI-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512VBMI-NEXT: vmovaps {{[0-9]+}}(%rsp), %xmm0
-; AVX512VBMI-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
-; AVX512VBMI-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: vmovaps %xmm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r9, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %r8, -{{[0-9]+}}(%rsp)
@@ -2562,6 +2555,9 @@ define i64 @lshr_extract_i512_i64(i512 %a0, i512 %a1) nounwind {
; AVX512VBMI-NEXT: movq %rdx, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rsi, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movq %rdi, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; AVX512VBMI-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; AVX512VBMI-NEXT: movl %r10d, %ecx
; AVX512VBMI-NEXT: shrl $3, %ecx
; AVX512VBMI-NEXT: andl $56, %ecx
diff --git a/llvm/test/CodeGen/X86/shuffle-half.ll b/llvm/test/CodeGen/X86/shuffle-half.ll
index cc7bfb58c329b..eb4577b2a46c9 100644
--- a/llvm/test/CodeGen/X86/shuffle-half.ll
+++ b/llvm/test/CodeGen/X86/shuffle-half.ll
@@ -4,12 +4,13 @@
define <32 x half> @dump_vec() {
; CHECK-LABEL: dump_vec:
; CHECK: # %bb.0:
-; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0
; CHECK-NEXT: testb %al, %al
-; CHECK-NEXT: jne .LBB0_2
-; CHECK-NEXT: # %bb.1: # %cond.load
+; CHECK-NEXT: jne .LBB0_1
+; CHECK-NEXT: # %bb.3: # %cond.load
; CHECK-NEXT: vmovups (%rax), %zmm0
-; CHECK-NEXT: .LBB0_2:
+; CHECK-NEXT: retq
+; CHECK-NEXT: .LBB0_1:
+; CHECK-NEXT: vxorps %xmm0, %xmm0, %xmm0
; CHECK-NEXT: retq
%1 = call <32 x half> @llvm.masked.load.v32f16.p0(ptr poison, i32 2, <32 x i1> poison, <32 x half> <half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0, half 0.0>)
ret <32 x half> %1
diff --git a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
index 05c855ed90b3f..b5f98dbcbd70f 100644
--- a/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
+++ b/llvm/test/CodeGen/X86/shuffle-vs-trunc-256.ll
@@ -745,9 +745,8 @@ define <2 x i64> @trunc_v4i64_to_v4i16_return_v2i64(<4 x i64> %vec) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
@@ -948,9 +947,8 @@ define <8 x i16> @trunc_v4i64_to_v4i16_return_v8i16(<4 x i64> %vec) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
; AVX2-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/speculative-load-hardening-gather.ll b/llvm/test/CodeGen/X86/speculative-load-hardening-gather.ll
index 7b3667420ec6d..ee8f6f566b2f4 100644
--- a/llvm/test/CodeGen/X86/speculative-load-hardening-gather.ll
+++ b/llvm/test/CodeGen/X86/speculative-load-hardening-gather.ll
@@ -379,8 +379,8 @@ define <16 x float> @test_llvm_x86_avx512_gather_dps_512(ptr %b, <16 x i32> %iv)
; CHECK-NEXT: movq %rsp, %rax
; CHECK-NEXT: movq $-1, %rcx
; CHECK-NEXT: sarq $63, %rax
-; CHECK-NEXT: kxnorw %k0, %k0, %k1
; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: kxnorw %k0, %k0, %k1
; CHECK-NEXT: orq %rax, %rdi
; CHECK-NEXT: vpbroadcastq %rax, %zmm2
; CHECK-NEXT: vporq %zmm0, %zmm2, %zmm0
@@ -472,8 +472,8 @@ define <16 x i32> @test_llvm_x86_avx512_gather_dpi_512(ptr %b, <16 x i32> %iv) #
; CHECK-NEXT: movq %rsp, %rax
; CHECK-NEXT: movq $-1, %rcx
; CHECK-NEXT: sarq $63, %rax
-; CHECK-NEXT: kxnorw %k0, %k0, %k1
; CHECK-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; CHECK-NEXT: kxnorw %k0, %k0, %k1
; CHECK-NEXT: orq %rax, %rdi
; CHECK-NEXT: vpbroadcastq %rax, %zmm2
; CHECK-NEXT: vporq %zmm0, %zmm2, %zmm0
diff --git a/llvm/test/CodeGen/X86/vec_int_to_fp.ll b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
index 1dde6c3bff448..cf8b5d2bb39eb 100644
--- a/llvm/test/CodeGen/X86/vec_int_to_fp.ll
+++ b/llvm/test/CodeGen/X86/vec_int_to_fp.ll
@@ -960,30 +960,30 @@ define <4 x double> @uitofp_4i64_to_4f64(<4 x i64> %a) {
;
; AVX2-LABEL: uitofp_4i64_to_4f64:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
-; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
-; AVX2-NEXT: vpor %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpsrlq $32, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlq $32, %ymm0, %ymm1
; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072]
-; AVX2-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpor %ymm2, %ymm1, %ymm1
; AVX2-NEXT: vbroadcastsd {{.*#+}} ymm2 = [1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25]
-; AVX2-NEXT: vsubpd %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vaddpd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vsubpd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vxorpd %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7]
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm2 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
+; AVX2-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; AVX2-NEXT: retq
;
; AVX512F-LABEL: uitofp_4i64_to_4f64:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
-; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm2 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
-; AVX512F-NEXT: vpor %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpsrlq $32, %ymm0, %ymm0
+; AVX512F-NEXT: vpsrlq $32, %ymm0, %ymm1
; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm2 = [4985484787499139072,4985484787499139072,4985484787499139072,4985484787499139072]
-; AVX512F-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT: vpor %ymm2, %ymm1, %ymm1
; AVX512F-NEXT: vbroadcastsd {{.*#+}} ymm2 = [1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25,1.9342813118337666E+25]
-; AVX512F-NEXT: vsubpd %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT: vaddpd %ymm0, %ymm1, %ymm0
+; AVX512F-NEXT: vsubpd %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT: vxorpd %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[2],ymm2[3],ymm0[4],ymm2[5],ymm0[6],ymm2[7]
+; AVX512F-NEXT: vpbroadcastq {{.*#+}} ymm2 = [4841369599423283200,4841369599423283200,4841369599423283200,4841369599423283200]
+; AVX512F-NEXT: vpor %ymm2, %ymm0, %ymm0
+; AVX512F-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: retq
;
; AVX512VL-LABEL: uitofp_4i64_to_4f64:
diff --git a/llvm/test/CodeGen/X86/vec_saddo.ll b/llvm/test/CodeGen/X86/vec_saddo.ll
index 78dd2cf783ef8..33a74f94dd6f3 100644
--- a/llvm/test/CodeGen/X86/vec_saddo.ll
+++ b/llvm/test/CodeGen/X86/vec_saddo.ll
@@ -325,28 +325,28 @@ define <6 x i32> @saddo_v6i32(<6 x i32> %a0, <6 x i32> %a1, ptr %p2) nounwind {
;
; AVX2-LABEL: saddo_v6i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpcmpgtd %ymm1, %ymm2, %ymm2
-; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm1
-; AVX2-NEXT: vpcmpgtd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpxor %ymm0, %ymm2, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX2-NEXT: vmovq %xmm2, 16(%rdi)
-; AVX2-NEXT: vmovdqa %xmm1, (%rdi)
+; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm2
+; AVX2-NEXT: vpcmpgtd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpcmpgtd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: vpxor %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm1
+; AVX2-NEXT: vmovq %xmm1, 16(%rdi)
+; AVX2-NEXT: vmovdqa %xmm2, (%rdi)
; AVX2-NEXT: retq
;
; AVX512-LABEL: saddo_v6i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpcmpgtd %ymm1, %ymm2, %k0
-; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: vpcmpgtd %ymm2, %ymm0, %k0
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512-NEXT: vpcmpgtd %ymm1, %ymm0, %k1
-; AVX512-NEXT: kxorw %k1, %k0, %k1
+; AVX512-NEXT: kxorw %k0, %k1, %k1
; AVX512-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX512-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}
-; AVX512-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX512-NEXT: vmovq %xmm2, 16(%rdi)
-; AVX512-NEXT: vmovdqa %xmm1, (%rdi)
+; AVX512-NEXT: vextracti128 $1, %ymm2, %xmm1
+; AVX512-NEXT: vmovq %xmm1, 16(%rdi)
+; AVX512-NEXT: vmovdqa %xmm2, (%rdi)
; AVX512-NEXT: retq
%t = call {<6 x i32>, <6 x i1>} @llvm.sadd.with.overflow.v6i32(<6 x i32> %a0, <6 x i32> %a1)
%val = extractvalue {<6 x i32>, <6 x i1>} %t, 0
@@ -393,24 +393,24 @@ define <8 x i32> @saddo_v8i32(<8 x i32> %a0, <8 x i32> %a1, ptr %p2) nounwind {
;
; AVX2-LABEL: saddo_v8i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpcmpgtd %ymm1, %ymm2, %ymm2
-; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm1
-; AVX2-NEXT: vpcmpgtd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpxor %ymm0, %ymm2, %ymm0
-; AVX2-NEXT: vmovdqa %ymm1, (%rdi)
+; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm2
+; AVX2-NEXT: vpcmpgtd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpcmpgtd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: vpxor %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vmovdqa %ymm2, (%rdi)
; AVX2-NEXT: retq
;
; AVX512-LABEL: saddo_v8i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpcmpgtd %ymm1, %ymm2, %k0
-; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm1
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm2
+; AVX512-NEXT: vpcmpgtd %ymm2, %ymm0, %k0
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512-NEXT: vpcmpgtd %ymm1, %ymm0, %k1
-; AVX512-NEXT: kxorw %k1, %k0, %k1
+; AVX512-NEXT: kxorw %k0, %k1, %k1
; AVX512-NEXT: vpcmpeqd %ymm0, %ymm0, %ymm0
; AVX512-NEXT: vmovdqa32 %ymm0, %ymm0 {%k1} {z}
-; AVX512-NEXT: vmovdqa %ymm1, (%rdi)
+; AVX512-NEXT: vmovdqa %ymm2, (%rdi)
; AVX512-NEXT: retq
%t = call {<8 x i32>, <8 x i1>} @llvm.sadd.with.overflow.v8i32(<8 x i32> %a0, <8 x i32> %a1)
%val = extractvalue {<8 x i32>, <8 x i1>} %t, 0
@@ -492,33 +492,33 @@ define <16 x i32> @saddo_v16i32(<16 x i32> %a0, <16 x i32> %a1, ptr %p2) nounwin
;
; AVX2-LABEL: saddo_v16i32:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT: vpcmpgtd %ymm3, %ymm4, %ymm5
-; AVX2-NEXT: vpaddd %ymm3, %ymm1, %ymm3
-; AVX2-NEXT: vpcmpgtd %ymm3, %ymm1, %ymm1
-; AVX2-NEXT: vpxor %ymm1, %ymm5, %ymm1
-; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX2-NEXT: vpackssdw %xmm5, %xmm1, %xmm1
-; AVX2-NEXT: vpcmpgtd %ymm2, %ymm4, %ymm4
-; AVX2-NEXT: vpaddd %ymm2, %ymm0, %ymm2
-; AVX2-NEXT: vpcmpgtd %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpxor %ymm0, %ymm4, %ymm0
+; AVX2-NEXT: vpaddd %ymm3, %ymm1, %ymm4
+; AVX2-NEXT: vpcmpgtd %ymm4, %ymm1, %ymm1
+; AVX2-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX2-NEXT: vpcmpgtd %ymm3, %ymm5, %ymm3
+; AVX2-NEXT: vpxor %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm3
+; AVX2-NEXT: vpackssdw %xmm3, %xmm1, %xmm1
+; AVX2-NEXT: vpaddd %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpcmpgtd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpcmpgtd %ymm2, %ymm5, %ymm2
+; AVX2-NEXT: vpxor %ymm0, %ymm2, %ymm0
; AVX2-NEXT: vpacksswb %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpmovsxbd %xmm1, %ymm1
; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0,0,0,0,4,4,4,4,8,8,8,8,12,12,12,12,16,16,16,16,20,20,20,20,24,24,24,24,28,28,28,28]
-; AVX2-NEXT: vmovdqa %ymm3, 32(%rdi)
-; AVX2-NEXT: vmovdqa %ymm2, (%rdi)
+; AVX2-NEXT: vmovdqa %ymm4, 32(%rdi)
+; AVX2-NEXT: vmovdqa %ymm3, (%rdi)
; AVX2-NEXT: retq
;
; AVX512-LABEL: saddo_v16i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512-NEXT: vpcmpgtd %zmm1, %zmm2, %k0
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm1
+; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm2
+; AVX512-NEXT: vpcmpgtd %zmm2, %zmm0, %k0
+; AVX512-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512-NEXT: vpcmpgtd %zmm1, %zmm0, %k1
-; AVX512-NEXT: kxorw %k1, %k0, %k1
+; AVX512-NEXT: kxorw %k0, %k1, %k1
; AVX512-NEXT: vpternlogd {{.*#+}} zmm0 {%k1} {z} = -1
-; AVX512-NEXT: vmovdqa64 %zmm1, (%rdi)
+; AVX512-NEXT: vmovdqa64 %zmm2, (%rdi)
; AVX512-NEXT: retq
%t = call {<16 x i32>, <16 x i1>} @llvm.sadd.with.overflow.v16i32(<16 x i32> %a0, <16 x i32> %a1)
%val = extractvalue {<16 x i32>, <16 x i1>} %t, 0
diff --git a/llvm/test/CodeGen/X86/vector-extend-inreg.ll b/llvm/test/CodeGen/X86/vector-extend-inreg.ll
index 7630255d3d086..9648eb5fc584a 100644
--- a/llvm/test/CodeGen/X86/vector-extend-inreg.ll
+++ b/llvm/test/CodeGen/X86/vector-extend-inreg.ll
@@ -70,12 +70,12 @@ define i64 @extract_any_extend_vector_inreg_v16i64(<16 x i64> %a0, i32 %a1) noun
; X86-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
; X86-AVX-NEXT: vmovaps %ymm1, {{[0-9]+}}(%esp)
; X86-AVX-NEXT: vmovaps %ymm1, {{[0-9]+}}(%esp)
-; X86-AVX-NEXT: vmovaps %ymm1, {{[0-9]+}}(%esp)
; X86-AVX-NEXT: vmovaps %ymm0, {{[0-9]+}}(%esp)
; X86-AVX-NEXT: vmovaps %ymm1, {{[0-9]+}}(%esp)
; X86-AVX-NEXT: vmovaps %ymm1, {{[0-9]+}}(%esp)
-; X86-AVX-NEXT: vmovaps %ymm1, (%esp)
+; X86-AVX-NEXT: vmovaps %ymm1, {{[0-9]+}}(%esp)
; X86-AVX-NEXT: vmovaps %ymm0, {{[0-9]+}}(%esp)
+; X86-AVX-NEXT: vmovaps %ymm1, (%esp)
; X86-AVX-NEXT: leal 1(%eax,%eax), %ecx
; X86-AVX-NEXT: andl $15, %eax
; X86-AVX-NEXT: movl 128(%esp,%eax,8), %eax
diff --git a/llvm/test/CodeGen/X86/vector-fshl-256.ll b/llvm/test/CodeGen/X86/vector-fshl-256.ll
index a56e22d9f1f3f..dde3ac499cd4c 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-256.ll
@@ -356,8 +356,8 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %
; AVX2-LABEL: var_funnnel_v16i16:
; AVX2: # %bb.0:
; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15]
; AVX2-NEXT: vpsllvd %ymm5, %ymm3, %ymm3
; AVX2-NEXT: vpsrld $16, %ymm3, %ymm3
diff --git a/llvm/test/CodeGen/X86/vector-fshl-512.ll b/llvm/test/CodeGen/X86/vector-fshl-512.ll
index 9d35202209533..41355d8875ee4 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-512.ll
@@ -357,8 +357,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
; AVX512BW-LABEL: var_funnnel_v64i8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512BW-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]
; AVX512BW-NEXT: vpsllvw %zmm5, %zmm3, %zmm3
; AVX512BW-NEXT: vpsrlw $8, %zmm3, %zmm3
@@ -372,8 +372,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
; AVX512VBMI2-LABEL: var_funnnel_v64i8:
; AVX512VBMI2: # %bb.0:
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512VBMI2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]
; AVX512VBMI2-NEXT: vpsllvw %zmm5, %zmm3, %zmm3
; AVX512VBMI2-NEXT: vpsrlw $8, %zmm3, %zmm3
@@ -387,8 +387,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
; AVX512VLBW-LABEL: var_funnnel_v64i8:
; AVX512VLBW: # %bb.0:
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512VLBW-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]
; AVX512VLBW-NEXT: vpsllvw %zmm5, %zmm3, %zmm3
; AVX512VLBW-NEXT: vpsrlw $8, %zmm3, %zmm3
@@ -402,8 +402,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
; AVX512VLVBMI2-LABEL: var_funnnel_v64i8:
; AVX512VLVBMI2: # %bb.0:
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512VLVBMI2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]
; AVX512VLVBMI2-NEXT: vpsllvw %zmm5, %zmm3, %zmm3
; AVX512VLVBMI2-NEXT: vpsrlw $8, %zmm3, %zmm3
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
index eb818205a0011..f90a5106ed163 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-256.ll
@@ -267,8 +267,8 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
;
; AVX2-LABEL: var_funnnel_v16i16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
; AVX2-NEXT: vpsllvd %ymm3, %ymm4, %ymm3
@@ -282,8 +282,8 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
;
; AVX512F-LABEL: var_funnnel_v16i16:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512F-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
; AVX512F-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
; AVX512F-NEXT: vpsllvd %ymm3, %ymm4, %ymm3
@@ -297,8 +297,8 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
;
; AVX512VL-LABEL: var_funnnel_v16i16:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VL-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
; AVX512VL-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
; AVX512VL-NEXT: vpsllvd %ymm3, %ymm4, %ymm3
@@ -483,8 +483,8 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
; AVX512BW-LABEL: var_funnnel_v32i8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512BW-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]
; AVX512BW-NEXT: vpsllvw %zmm4, %zmm2, %zmm2
; AVX512BW-NEXT: vpsrlw $8, %ymm2, %ymm2
@@ -497,8 +497,8 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
;
; AVX512VLBW-LABEL: var_funnnel_v32i8:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; AVX512VLBW-NEXT: vpsllvw %ymm3, %ymm4, %ymm3
@@ -513,8 +513,8 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
; AVX512VBMI2-LABEL: var_funnnel_v32i8:
; AVX512VBMI2: # %bb.0:
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512VBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VBMI2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]
; AVX512VBMI2-NEXT: vpsllvw %zmm4, %zmm2, %zmm2
; AVX512VBMI2-NEXT: vpsrlw $8, %ymm2, %ymm2
@@ -527,8 +527,8 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
;
; AVX512VLVBMI2-LABEL: var_funnnel_v32i8:
; AVX512VLVBMI2: # %bb.0:
-; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VLVBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; AVX512VLVBMI2-NEXT: vpsllvw %ymm3, %ymm4, %ymm3
diff --git a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
index fdf1970e6298b..496f19d45bb01 100644
--- a/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshl-rot-512.ll
@@ -216,8 +216,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
;
; AVX512BW-LABEL: var_funnnel_v64i8:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; AVX512BW-NEXT: vpsllvw %zmm3, %zmm4, %zmm3
@@ -231,8 +231,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
;
; AVX512VLBW-LABEL: var_funnnel_v64i8:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; AVX512VLBW-NEXT: vpsllvw %zmm3, %zmm4, %zmm3
@@ -246,8 +246,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
;
; AVX512VBMI2-LABEL: var_funnnel_v64i8:
; AVX512VBMI2: # %bb.0:
-; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; AVX512VBMI2-NEXT: vpsllvw %zmm3, %zmm4, %zmm3
@@ -261,8 +261,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
;
; AVX512VLVBMI2-LABEL: var_funnnel_v64i8:
; AVX512VLVBMI2: # %bb.0:
-; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VLVBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; AVX512VLVBMI2-NEXT: vpsllvw %zmm3, %zmm4, %zmm3
diff --git a/llvm/test/CodeGen/X86/vector-fshr-256.ll b/llvm/test/CodeGen/X86/vector-fshr-256.ll
index 425441b738399..0b8647ba10575 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-256.ll
@@ -386,8 +386,8 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %y, <16 x i16> %
; AVX2-LABEL: var_funnnel_v16i16:
; AVX2: # %bb.0:
; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm2, %ymm2
; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm2[4],ymm4[4],ymm2[5],ymm4[5],ymm2[6],ymm4[6],ymm2[7],ymm4[7],ymm2[12],ymm4[12],ymm2[13],ymm4[13],ymm2[14],ymm4[14],ymm2[15],ymm4[15]
; AVX2-NEXT: vpsrlvd %ymm5, %ymm3, %ymm3
; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm3[0],ymm4[1],ymm3[2],ymm4[3],ymm3[4],ymm4[5],ymm3[6],ymm4[7],ymm3[8],ymm4[9],ymm3[10],ymm4[11],ymm3[12],ymm4[13],ymm3[14],ymm4[15]
diff --git a/llvm/test/CodeGen/X86/vector-fshr-512.ll b/llvm/test/CodeGen/X86/vector-fshr-512.ll
index 4c6cd5feaf593..9362fcadd43d6 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-512.ll
@@ -355,8 +355,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
; AVX512BW-LABEL: var_funnnel_v64i8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512BW-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]
; AVX512BW-NEXT: vpsrlvw %zmm5, %zmm3, %zmm3
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} zmm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
@@ -371,8 +371,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
; AVX512VBMI2-LABEL: var_funnnel_v64i8:
; AVX512VBMI2: # %bb.0:
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512VBMI2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]
; AVX512VBMI2-NEXT: vpsrlvw %zmm5, %zmm3, %zmm3
; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
@@ -385,8 +385,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
; AVX512VLBW-LABEL: var_funnnel_v64i8:
; AVX512VLBW: # %bb.0:
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512VLBW-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]
; AVX512VLBW-NEXT: vpsrlvw %zmm5, %zmm3, %zmm3
; AVX512VLBW-NEXT: vpbroadcastw {{.*#+}} zmm5 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
@@ -401,8 +401,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y, <64 x i8> %amt)
; AVX512VLVBMI2-LABEL: var_funnnel_v64i8:
; AVX512VLVBMI2: # %bb.0:
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm0[8],zmm1[9],zmm0[9],zmm1[10],zmm0[10],zmm1[11],zmm0[11],zmm1[12],zmm0[12],zmm1[13],zmm0[13],zmm1[14],zmm0[14],zmm1[15],zmm0[15],zmm1[24],zmm0[24],zmm1[25],zmm0[25],zmm1[26],zmm0[26],zmm1[27],zmm0[27],zmm1[28],zmm0[28],zmm1[29],zmm0[29],zmm1[30],zmm0[30],zmm1[31],zmm0[31],zmm1[40],zmm0[40],zmm1[41],zmm0[41],zmm1[42],zmm0[42],zmm1[43],zmm0[43],zmm1[44],zmm0[44],zmm1[45],zmm0[45],zmm1[46],zmm0[46],zmm1[47],zmm0[47],zmm1[56],zmm0[56],zmm1[57],zmm0[57],zmm1[58],zmm0[58],zmm1[59],zmm0[59],zmm1[60],zmm0[60],zmm1[61],zmm0[61],zmm1[62],zmm0[62],zmm1[63],zmm0[63]
-; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512VLVBMI2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm2, %zmm2
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm5 = zmm2[8],zmm4[8],zmm2[9],zmm4[9],zmm2[10],zmm4[10],zmm2[11],zmm4[11],zmm2[12],zmm4[12],zmm2[13],zmm4[13],zmm2[14],zmm4[14],zmm2[15],zmm4[15],zmm2[24],zmm4[24],zmm2[25],zmm4[25],zmm2[26],zmm4[26],zmm2[27],zmm4[27],zmm2[28],zmm4[28],zmm2[29],zmm4[29],zmm2[30],zmm4[30],zmm2[31],zmm4[31],zmm2[40],zmm4[40],zmm2[41],zmm4[41],zmm2[42],zmm4[42],zmm2[43],zmm4[43],zmm2[44],zmm4[44],zmm2[45],zmm4[45],zmm2[46],zmm4[46],zmm2[47],zmm4[47],zmm2[56],zmm4[56],zmm2[57],zmm4[57],zmm2[58],zmm4[58],zmm2[59],zmm4[59],zmm2[60],zmm4[60],zmm2[61],zmm4[61],zmm2[62],zmm4[62],zmm2[63],zmm4[63]
; AVX512VLVBMI2-NEXT: vpsrlvw %zmm5, %zmm3, %zmm3
; AVX512VLVBMI2-NEXT: vpunpcklbw {{.*#+}} zmm0 = zmm1[0],zmm0[0],zmm1[1],zmm0[1],zmm1[2],zmm0[2],zmm1[3],zmm0[3],zmm1[4],zmm0[4],zmm1[5],zmm0[5],zmm1[6],zmm0[6],zmm1[7],zmm0[7],zmm1[16],zmm0[16],zmm1[17],zmm0[17],zmm1[18],zmm0[18],zmm1[19],zmm0[19],zmm1[20],zmm0[20],zmm1[21],zmm0[21],zmm1[22],zmm0[22],zmm1[23],zmm0[23],zmm1[32],zmm0[32],zmm1[33],zmm0[33],zmm1[34],zmm0[34],zmm1[35],zmm0[35],zmm1[36],zmm0[36],zmm1[37],zmm0[37],zmm1[38],zmm0[38],zmm1[39],zmm0[39],zmm1[48],zmm0[48],zmm1[49],zmm0[49],zmm1[50],zmm0[50],zmm1[51],zmm0[51],zmm1[52],zmm0[52],zmm1[53],zmm0[53],zmm1[54],zmm0[54],zmm1[55],zmm0[55]
@@ -884,27 +884,27 @@ define <32 x i16> @constant_funnnel_v32i16(<32 x i16> %x, <32 x i16> %y) nounwin
define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
; AVX512F-LABEL: constant_funnnel_v64i8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
+; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15],ymm3[24],ymm2[24],ymm3[25],ymm2[25],ymm3[26],ymm2[26],ymm3[27],ymm2[27],ymm3[28],ymm2[28],ymm3[29],ymm2[29],ymm3[30],ymm2[30],ymm3[31],ymm2[31]
; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
; AVX512F-NEXT: # ymm5 = mem[0,1,0,1]
; AVX512F-NEXT: vpmullw %ymm5, %ymm4, %ymm4
; AVX512F-NEXT: vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23]
+; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[16],ymm2[16],ymm3[17],ymm2[17],ymm3[18],ymm2[18],ymm3[19],ymm2[19],ymm3[20],ymm2[20],ymm3[21],ymm2[21],ymm3[22],ymm2[22],ymm3[23],ymm2[23]
; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
; AVX512F-NEXT: # ymm6 = mem[0,1,0,1]
-; AVX512F-NEXT: vpmullw %ymm6, %ymm2, %ymm2
-; AVX512F-NEXT: vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT: vpackuswb %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]
+; AVX512F-NEXT: vpmullw %ymm6, %ymm3, %ymm3
+; AVX512F-NEXT: vpsrlw $8, %ymm3, %ymm3
+; AVX512F-NEXT: vpackuswb %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; AVX512F-NEXT: vpmullw %ymm5, %ymm4, %ymm4
; AVX512F-NEXT: vpsrlw $8, %ymm4, %ymm4
-; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[4],ymm3[4],ymm1[5],ymm3[5],ymm1[6],ymm3[6],ymm1[7],ymm3[7],ymm1[16],ymm3[16],ymm1[17],ymm3[17],ymm1[18],ymm3[18],ymm1[19],ymm3[19],ymm1[20],ymm3[20],ymm1[21],ymm3[21],ymm1[22],ymm3[22],ymm1[23],ymm3[23]
+; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
; AVX512F-NEXT: vpmullw %ymm6, %ymm1, %ymm1
; AVX512F-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512F-NEXT: vpackuswb %ymm4, %ymm1, %ymm1
-; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
; AVX512F-NEXT: vpaddb %ymm0, %ymm0, %ymm2
; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,64,0,16,0,4,0,1,0,1,0,4,0,16,0,64,0,64,0,16,0,4,0,1,0,1,0,4,0,16,0,64]
; AVX512F-NEXT: # ymm3 = mem[0,1,0,1]
@@ -926,27 +926,27 @@ define <64 x i8> @constant_funnnel_v64i8(<64 x i8> %x, <64 x i8> %y) nounwind {
;
; AVX512VL-LABEL: constant_funnnel_v64i8:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm2[8],ymm3[8],ymm2[9],ymm3[9],ymm2[10],ymm3[10],ymm2[11],ymm3[11],ymm2[12],ymm3[12],ymm2[13],ymm3[13],ymm2[14],ymm3[14],ymm2[15],ymm3[15],ymm2[24],ymm3[24],ymm2[25],ymm3[25],ymm2[26],ymm3[26],ymm2[27],ymm3[27],ymm2[28],ymm3[28],ymm2[29],ymm3[29],ymm2[30],ymm3[30],ymm2[31],ymm3[31]
+; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm3
+; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm3[8],ymm2[8],ymm3[9],ymm2[9],ymm3[10],ymm2[10],ymm3[11],ymm2[11],ymm3[12],ymm2[12],ymm3[13],ymm2[13],ymm3[14],ymm2[14],ymm3[15],ymm2[15],ymm3[24],ymm2[24],ymm3[25],ymm2[25],ymm3[26],ymm2[26],ymm3[27],ymm2[27],ymm3[28],ymm2[28],ymm3[29],ymm2[29],ymm3[30],ymm2[30],ymm3[31],ymm2[31]
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [256,2,4,8,16,32,64,128,256,2,4,8,16,32,64,128]
; AVX512VL-NEXT: # ymm5 = mem[0,1,0,1]
; AVX512VL-NEXT: vpmullw %ymm5, %ymm4, %ymm4
; AVX512VL-NEXT: vpsrlw $8, %ymm4, %ymm4
-; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm3[0],ymm2[1],ymm3[1],ymm2[2],ymm3[2],ymm2[3],ymm3[3],ymm2[4],ymm3[4],ymm2[5],ymm3[5],ymm2[6],ymm3[6],ymm2[7],ymm3[7],ymm2[16],ymm3[16],ymm2[17],ymm3[17],ymm2[18],ymm3[18],ymm2[19],ymm3[19],ymm2[20],ymm3[20],ymm2[21],ymm3[21],ymm2[22],ymm3[22],ymm2[23],ymm3[23]
+; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm3 = ymm3[0],ymm2[0],ymm3[1],ymm2[1],ymm3[2],ymm2[2],ymm3[3],ymm2[3],ymm3[4],ymm2[4],ymm3[5],ymm2[5],ymm3[6],ymm2[6],ymm3[7],ymm2[7],ymm3[16],ymm2[16],ymm3[17],ymm2[17],ymm3[18],ymm2[18],ymm3[19],ymm2[19],ymm3[20],ymm2[20],ymm3[21],ymm2[21],ymm3[22],ymm2[22],ymm3[23],ymm2[23]
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
; AVX512VL-NEXT: # ymm6 = mem[0,1,0,1]
-; AVX512VL-NEXT: vpmullw %ymm6, %ymm2, %ymm2
-; AVX512VL-NEXT: vpsrlw $8, %ymm2, %ymm2
-; AVX512VL-NEXT: vpackuswb %ymm4, %ymm2, %ymm2
-; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]
+; AVX512VL-NEXT: vpmullw %ymm6, %ymm3, %ymm3
+; AVX512VL-NEXT: vpsrlw $8, %ymm3, %ymm3
+; AVX512VL-NEXT: vpackuswb %ymm4, %ymm3, %ymm3
+; AVX512VL-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; AVX512VL-NEXT: vpmullw %ymm5, %ymm4, %ymm4
; AVX512VL-NEXT: vpsrlw $8, %ymm4, %ymm4
-; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[4],ymm3[4],ymm1[5],ymm3[5],ymm1[6],ymm3[6],ymm1[7],ymm3[7],ymm1[16],ymm3[16],ymm1[17],ymm3[17],ymm1[18],ymm3[18],ymm1[19],ymm3[19],ymm1[20],ymm3[20],ymm1[21],ymm3[21],ymm1[22],ymm3[22],ymm1[23],ymm3[23]
+; AVX512VL-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
; AVX512VL-NEXT: vpmullw %ymm6, %ymm1, %ymm1
; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512VL-NEXT: vpackuswb %ymm4, %ymm1, %ymm1
-; AVX512VL-NEXT: vinserti64x4 $1, %ymm2, %zmm1, %zmm1
+; AVX512VL-NEXT: vinserti64x4 $1, %ymm3, %zmm1, %zmm1
; AVX512VL-NEXT: vpaddb %ymm0, %ymm0, %ymm2
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,64,0,16,0,4,0,1,0,1,0,4,0,16,0,64,0,64,0,16,0,4,0,1,0,1,0,4,0,16,0,64]
; AVX512VL-NEXT: # ymm3 = mem[0,1,0,1]
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
index ca5e06e59096d..4eb4247c7ee1c 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-256.ll
@@ -282,8 +282,8 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
;
; AVX2-LABEL: var_funnnel_v16i16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
; AVX2-NEXT: vpsrlvd %ymm3, %ymm4, %ymm3
@@ -297,8 +297,8 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
;
; AVX512F-LABEL: var_funnnel_v16i16:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512F-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
; AVX512F-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
; AVX512F-NEXT: vpsrlvd %ymm3, %ymm4, %ymm3
@@ -312,8 +312,8 @@ define <16 x i16> @var_funnnel_v16i16(<16 x i16> %x, <16 x i16> %amt) nounwind {
;
; AVX512VL-LABEL: var_funnnel_v16i16:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VL-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
; AVX512VL-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
; AVX512VL-NEXT: vpsrlvd %ymm3, %ymm4, %ymm3
@@ -502,8 +502,8 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
; AVX512BW-LABEL: var_funnnel_v32i8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512BW-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]
; AVX512BW-NEXT: vpsrlvw %zmm4, %zmm2, %zmm2
; AVX512BW-NEXT: vpbroadcastw {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255,255,255,255,255,255,255,255,255]
@@ -517,8 +517,8 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
;
; AVX512VLBW-LABEL: var_funnnel_v32i8:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; AVX512VLBW-NEXT: vpsrlvw %ymm3, %ymm4, %ymm3
@@ -534,8 +534,8 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
; AVX512VBMI2-LABEL: var_funnnel_v32i8:
; AVX512VBMI2: # %bb.0:
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512VBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VBMI2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]
; AVX512VBMI2-NEXT: vpsrlvw %zmm4, %zmm2, %zmm2
; AVX512VBMI2-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7,16,16,17,17,18,18,19,19,20,20,21,21,22,22,23,23]
@@ -548,8 +548,8 @@ define <32 x i8> @var_funnnel_v32i8(<32 x i8> %x, <32 x i8> %amt) nounwind {
;
; AVX512VLVBMI2-LABEL: var_funnnel_v32i8:
; AVX512VLVBMI2: # %bb.0:
-; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VLVBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; AVX512VLVBMI2-NEXT: vpsrlvw %ymm3, %ymm4, %ymm3
diff --git a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
index 27de093e5a985..fa98011399712 100644
--- a/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
+++ b/llvm/test/CodeGen/X86/vector-fshr-rot-512.ll
@@ -208,8 +208,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
;
; AVX512BW-LABEL: var_funnnel_v64i8:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; AVX512BW-NEXT: vpsrlvw %zmm3, %zmm4, %zmm3
@@ -224,8 +224,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
;
; AVX512VLBW-LABEL: var_funnnel_v64i8:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; AVX512VLBW-NEXT: vpsrlvw %zmm3, %zmm4, %zmm3
@@ -240,8 +240,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
;
; AVX512VBMI2-LABEL: var_funnnel_v64i8:
; AVX512VBMI2: # %bb.0:
-; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; AVX512VBMI2-NEXT: vpsrlvw %zmm3, %zmm4, %zmm3
@@ -254,8 +254,8 @@ define <64 x i8> @var_funnnel_v64i8(<64 x i8> %x, <64 x i8> %amt) nounwind {
;
; AVX512VLVBMI2-LABEL: var_funnnel_v64i8:
; AVX512VLVBMI2: # %bb.0:
-; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VLVBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; AVX512VLVBMI2-NEXT: vpsrlvw %zmm3, %zmm4, %zmm3
diff --git a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
index 31d70041f9d0b..d11a37fc042f0 100644
--- a/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
+++ b/llvm/test/CodeGen/X86/vector-idiv-udiv-512.ll
@@ -145,8 +145,8 @@ define <64 x i8> @test_div7_64i8(<64 x i8> %a) nounwind {
define <64 x i8> @test_divconstant_64i8(<64 x i8> %a) nounwind {
; AVX512F-LABEL: test_divconstant_64i8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
; AVX512F-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [256,256,64,256,256,256,256,256,128,256,256,256,256,256,256,256]
; AVX512F-NEXT: vpsrlw $8, %ymm3, %ymm3
@@ -336,17 +336,17 @@ define <32 x i16> @test_rem7_32i16(<32 x i16> %a) nounwind {
define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
; AVX512F-LABEL: test_rem7_64i8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
; AVX512F-NEXT: vpbroadcastw {{.*#+}} ymm4 = [37,0,37,0,37,0,37,0,37,0,37,0,37,0,37,0,37,0,37,0,37,0,37,0,37,0,37,0,37,0,37,0]
; AVX512F-NEXT: vpmullw %ymm4, %ymm3, %ymm3
; AVX512F-NEXT: vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm5 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
+; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm5 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
; AVX512F-NEXT: vpmullw %ymm4, %ymm5, %ymm5
; AVX512F-NEXT: vpsrlw $8, %ymm5, %ymm5
; AVX512F-NEXT: vpackuswb %ymm3, %ymm5, %ymm3
-; AVX512F-NEXT: vpsubb %ymm3, %ymm1, %ymm5
+; AVX512F-NEXT: vpsubb %ymm3, %ymm2, %ymm5
; AVX512F-NEXT: vpsrlw $1, %ymm5, %ymm5
; AVX512F-NEXT: vpbroadcastb {{.*#+}} ymm6 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
; AVX512F-NEXT: vpand %ymm6, %ymm5, %ymm5
@@ -358,25 +358,25 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
; AVX512F-NEXT: vpbroadcastb {{.*#+}} ymm8 = [63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63,63]
; AVX512F-NEXT: vpand %ymm3, %ymm8, %ymm3
; AVX512F-NEXT: vpsubb %ymm5, %ymm3, %ymm3
-; AVX512F-NEXT: vpaddb %ymm3, %ymm1, %ymm1
-; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
+; AVX512F-NEXT: vpaddb %ymm3, %ymm2, %ymm2
+; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
; AVX512F-NEXT: vpmullw %ymm4, %ymm3, %ymm3
; AVX512F-NEXT: vpsrlw $8, %ymm3, %ymm3
-; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
-; AVX512F-NEXT: vpmullw %ymm4, %ymm2, %ymm2
-; AVX512F-NEXT: vpsrlw $8, %ymm2, %ymm2
-; AVX512F-NEXT: vpackuswb %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpsubb %ymm2, %ymm0, %ymm3
+; AVX512F-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
+; AVX512F-NEXT: vpmullw %ymm4, %ymm1, %ymm1
+; AVX512F-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX512F-NEXT: vpackuswb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT: vpsubb %ymm1, %ymm0, %ymm3
; AVX512F-NEXT: vpsrlw $1, %ymm3, %ymm3
; AVX512F-NEXT: vpand %ymm6, %ymm3, %ymm3
-; AVX512F-NEXT: vpaddb %ymm2, %ymm3, %ymm2
-; AVX512F-NEXT: vpaddw %ymm2, %ymm2, %ymm3
+; AVX512F-NEXT: vpaddb %ymm1, %ymm3, %ymm1
+; AVX512F-NEXT: vpaddw %ymm1, %ymm1, %ymm3
; AVX512F-NEXT: vpand %ymm7, %ymm3, %ymm3
-; AVX512F-NEXT: vpsrlw $2, %ymm2, %ymm2
-; AVX512F-NEXT: vpand %ymm2, %ymm8, %ymm2
-; AVX512F-NEXT: vpsubb %ymm3, %ymm2, %ymm2
-; AVX512F-NEXT: vpaddb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpsrlw $2, %ymm1, %ymm1
+; AVX512F-NEXT: vpand %ymm1, %ymm8, %ymm1
+; AVX512F-NEXT: vpsubb %ymm3, %ymm1, %ymm1
+; AVX512F-NEXT: vpaddb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: test_rem7_64i8:
@@ -412,8 +412,8 @@ define <64 x i8> @test_rem7_64i8(<64 x i8> %a) nounwind {
define <64 x i8> @test_remconstant_64i8(<64 x i8> %a) nounwind {
; AVX512F-LABEL: test_remconstant_64i8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
; AVX512F-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
; AVX512F-NEXT: vpmullw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3 # [256,256,64,256,256,256,256,256,128,256,256,256,256,256,256,256]
; AVX512F-NEXT: vpsrlw $8, %ymm3, %ymm3
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll
index b210033b39b99..ce9f73a79b7dd 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-4.ll
@@ -5237,11 +5237,11 @@ define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,2,2,3,0,2,4,6]
; AVX2-FCP-NEXT: vpermd %ymm4, %ymm0, %ymm3
; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29]
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm3
-; AVX2-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm5
-; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm5, %ymm5
+; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm14 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,16,17,20,21,24,25,28,29]
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm3, %ymm3
+; AVX2-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm4
+; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm4, %ymm5
; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm5[0,1,2,3,4,5],ymm3[6,7]
; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm3[4,5,6,7]
; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -5253,9 +5253,9 @@ define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX2-FCP-NEXT: vpackusdw %xmm5, %xmm3, %xmm3
; AVX2-FCP-NEXT: vpackusdw %xmm2, %xmm3, %xmm3
; AVX2-FCP-NEXT: vpermd %ymm6, %ymm0, %ymm2
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm5
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm2, %ymm5
; AVX2-FCP-NEXT: vpermd %ymm7, %ymm0, %ymm6
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm6, %ymm7
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm6, %ymm7
; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3,4,5],ymm5[6,7]
; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm5[4,5,6,7]
; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -5270,11 +5270,12 @@ define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm5, %xmm3
; AVX2-FCP-NEXT: vmovdqa 480(%rdi), %ymm5
; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm5
-; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm5, %ymm5
-; AVX2-FCP-NEXT: vpermd %ymm7, %ymm0, %ymm14
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm14, %ymm7
+; AVX2-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm4
+; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm4, %ymm5
+; AVX2-FCP-NEXT: vpermd %ymm7, %ymm0, %ymm4
+; AVX2-FCP-NEXT: vmovdqu %ymm4, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm4, %ymm7
; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm7[0,1,2,3,4,5],ymm5[6,7]
; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3],ymm5[4,5,6,7]
; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
@@ -5285,272 +5286,275 @@ define void @load_i16_stride4_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX2-FCP-NEXT: vextracti128 $1, %ymm1, %xmm5
; AVX2-FCP-NEXT: vpackusdw %xmm5, %xmm1, %xmm1
; AVX2-FCP-NEXT: vpackusdw %xmm3, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vmovdqa 192(%rdi), %ymm5
-; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vmovdqa 224(%rdi), %ymm3
+; AVX2-FCP-NEXT: vmovdqa 192(%rdi), %ymm3
; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpermd %ymm3, %ymm0, %ymm9
-; AVX2-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm3
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm9, %ymm0
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm3, %ymm5
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm5[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-FCP-NEXT: vmovdqa 224(%rdi), %ymm5
+; AVX2-FCP-NEXT: vmovdqu %ymm5, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FCP-NEXT: vpermd %ymm5, %ymm0, %ymm9
+; AVX2-FCP-NEXT: vpermd %ymm3, %ymm0, %ymm5
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm9, %ymm0
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm5, %ymm3
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3,4,5],ymm0[6,7]
; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX2-FCP-NEXT: vmovdqa 288(%rdi), %xmm1
; AVX2-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vmovdqa 304(%rdi), %xmm12
-; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]
-; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm12, %xmm0
-; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm1
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; AVX2-FCP-NEXT: vmovdqa 256(%rdi), %xmm10
-; AVX2-FCP-NEXT: vmovdqa %xmm10, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vmovdqa 272(%rdi), %xmm7
-; AVX2-FCP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
-; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm7, %xmm7
-; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm10, %xmm8
+; AVX2-FCP-NEXT: vmovdqa 304(%rdi), %xmm0
+; AVX2-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm10 = [0,1,2,3,2,3,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm1, %xmm3
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; AVX2-FCP-NEXT: vmovdqa 256(%rdi), %xmm0
+; AVX2-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FCP-NEXT: vmovdqa 272(%rdi), %xmm12
+; AVX2-FCP-NEXT: vmovdqa {{.*#+}} xmm4 = [2,3,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
+; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm12, %xmm7
+; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm0, %xmm8
; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm7[0,1],xmm1[2,3]
-; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm10 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm2, %ymm2
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm6, %ymm6
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm7[0,1],xmm3[2,3]
+; AVX2-FCP-NEXT: vmovdqa {{.*#+}} ymm13 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15,18,19,22,23,26,27,30,31,18,19,22,23,26,27,30,31]
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm2, %ymm2
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm6, %ymm6
; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm6[0,1,2,3,4,5],ymm2[6,7]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm2
-; AVX2-FCP-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm1, %xmm6
-; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm2, %xmm7
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FCP-NEXT: vmovdqa 32(%rdi), %xmm3
+; AVX2-FCP-NEXT: vmovdqa 48(%rdi), %xmm15
+; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm15, %xmm2
+; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm3, %xmm6
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
; AVX2-FCP-NEXT: vmovdqa (%rdi), %xmm7
-; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm15
-; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm15, %xmm11
-; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm7, %xmm13
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm11[0,1],xmm6[2,3]
-; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm11
-; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm13
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm11 = ymm13[0,1,2,3,4,5],ymm11[6,7]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm6[0,1,2,3],ymm11[4,5,6,7]
-; AVX2-FCP-NEXT: vmovdqu %ymm6, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vmovdqa 160(%rdi), %xmm8
-; AVX2-FCP-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vmovdqa 176(%rdi), %xmm6
-; AVX2-FCP-NEXT: vmovdqa %xmm6, (%rsp) # 16-byte Spill
-; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm6, %xmm6
-; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm8, %xmm11
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm11[0],xmm6[0],xmm11[1],xmm6[1]
-; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %xmm8
-; AVX2-FCP-NEXT: vmovdqa %xmm8, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vmovdqa 144(%rdi), %xmm11
-; AVX2-FCP-NEXT: vmovdqa %xmm11, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm11, %xmm11
-; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm8, %xmm13
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm11 = xmm13[0],xmm11[0],xmm13[1],xmm11[1]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm6 = xmm11[0,1],xmm6[2,3]
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm9, %ymm9
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm3, %ymm3
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm3[0,1,2,3,4,5],ymm9[6,7]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm6[0,1,2,3],ymm3[4,5,6,7]
-; AVX2-FCP-NEXT: vmovdqu %ymm3, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vmovdqa 416(%rdi), %xmm6
-; AVX2-FCP-NEXT: vmovdqa 432(%rdi), %xmm13
-; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm13, %xmm3
-; AVX2-FCP-NEXT: vpshufb %xmm5, %xmm6, %xmm5
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm9 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; AVX2-FCP-NEXT: vmovdqa 384(%rdi), %xmm5
-; AVX2-FCP-NEXT: vmovdqa 400(%rdi), %xmm3
-; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm3, %xmm11
-; AVX2-FCP-NEXT: vpshufb %xmm0, %xmm5, %xmm0
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm11[0],xmm0[1],xmm11[1]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm9[2,3]
-; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm8 # 32-byte Reload
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm9
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm14, %ymm11
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5],ymm9[6,7]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm9[4,5,6,7]
+; AVX2-FCP-NEXT: vmovdqa 16(%rdi), %xmm2
+; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm2, %xmm0
+; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm7, %xmm11
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm11[0],xmm0[0],xmm11[1],xmm0[1]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm6[2,3]
+; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm1, %ymm6
+; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm1, %ymm11
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm6 = ymm11[0,1,2,3,4,5],ymm6[6,7]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm6[4,5,6,7]
; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm0 = [1,3,2,3,1,3,5,7]
-; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
-; AVX2-FCP-NEXT: vmovdqu %ymm8, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm11 # 32-byte Folded Reload
-; AVX2-FCP-NEXT: vmovdqu %ymm11, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm8, %ymm9
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm11, %ymm11
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm9 = ymm11[0,1,2,3,4,5],ymm9[6,7]
-; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,1,2,3]
+; AVX2-FCP-NEXT: vmovdqa 160(%rdi), %xmm1
; AVX2-FCP-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,1,2,3]
-; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[0,1,2,0,4,5,6,7]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm15[3,1,2,3]
-; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm7[3,1,2,3]
-; AVX2-FCP-NEXT: vmovdqa %xmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[2,0,2,3,4,5,6,7]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[2,0,2,3,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm2[0,1],xmm1[2,3]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm9[4,5,6,7]
+; AVX2-FCP-NEXT: vmovdqa 176(%rdi), %xmm0
+; AVX2-FCP-NEXT: vmovdqa %xmm0, (%rsp) # 16-byte Spill
+; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm0, %xmm0
+; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm1, %xmm6
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm6[0],xmm0[0],xmm6[1],xmm0[1]
+; AVX2-FCP-NEXT: vmovdqa 128(%rdi), %xmm0
+; AVX2-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FCP-NEXT: vmovdqa 144(%rdi), %xmm6
+; AVX2-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm6, %xmm6
+; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm0, %xmm11
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm11[0],xmm6[0],xmm11[1],xmm6[1]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm6[0,1],xmm1[2,3]
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm9, %ymm6
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm5, %ymm5
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm5 = ymm5[0,1,2,3,4,5],ymm6[6,7]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm5[4,5,6,7]
+; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FCP-NEXT: vmovdqa 416(%rdi), %xmm8
+; AVX2-FCP-NEXT: vmovdqa 432(%rdi), %xmm9
+; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm9, %xmm0
+; AVX2-FCP-NEXT: vpshufb %xmm10, %xmm8, %xmm5
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm5[0],xmm0[0],xmm5[1],xmm0[1]
+; AVX2-FCP-NEXT: vmovdqa 384(%rdi), %xmm10
+; AVX2-FCP-NEXT: vmovdqa 400(%rdi), %xmm5
+; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm5, %xmm11
+; AVX2-FCP-NEXT: vpshufb %xmm4, %xmm10, %xmm1
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm11[0],xmm1[1],xmm11[1]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0,1],xmm0[2,3]
+; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm1, %ymm1
+; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm4, %ymm11
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FCP-NEXT: vpmovsxbd {{.*#+}} ymm6 = [1,3,2,3,1,3,5,7]
+; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm1 # 32-byte Folded Reload
; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
+; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm0 # 32-byte Folded Reload
+; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm1, %ymm1
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm0, %ymm11
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm11[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm0 = xmm15[3,1,2,3]
+; AVX2-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[3,1,2,3]
+; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[0,1,2,0,4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm11 = xmm4[0,1,2,0,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm11[0],xmm3[0],xmm11[1],xmm3[1]
+; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm0 = xmm2[3,1,2,3]
+; AVX2-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm4 = xmm7[3,1,2,3]
+; AVX2-FCP-NEXT: vmovdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[2,0,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm7 = xmm4[2,0,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
+; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm0 # 32-byte Folded Reload
+; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm2 # 32-byte Folded Reload
; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm1
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm2
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm0, %ymm1
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm2, %ymm2
; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm12[3,1,2,3]
-; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
-; AVX2-FCP-NEXT: # xmm14 = mem[3,1,2,3]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm7 = xmm14[0,1,2,0,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
+; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Folded Reload
+; AVX2-FCP-NEXT: # xmm0 = mem[3,1,2,3]
+; AVX2-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm11 # 16-byte Folded Reload
; AVX2-FCP-NEXT: # xmm11 = mem[3,1,2,3]
-; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm9 # 16-byte Folded Reload
-; AVX2-FCP-NEXT: # xmm9 = mem[3,1,2,3]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm12 = xmm11[2,0,2,3,4,5,6,7]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm15 = xmm9[2,0,2,3,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm12 = xmm15[0],xmm12[0],xmm15[1],xmm12[1]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm12[0,1],xmm2[2,3]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[0,1,2,0,4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm11[0,1,2,0,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm7 = xmm12[3,1,2,3]
+; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
+; AVX2-FCP-NEXT: # xmm15 = mem[3,1,2,3]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm7[2,0,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm12 = xmm15[2,0,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm12[0],xmm3[0],xmm12[1],xmm3[1]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
+; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm0 # 32-byte Folded Reload
+; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm2 # 32-byte Folded Reload
; AVX2-FCP-NEXT: vmovdqu %ymm2, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm1, %ymm1
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm2, %ymm2
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm0, %ymm1
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm2, %ymm2
; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm2 = xmm13[3,1,2,3]
-; AVX2-FCP-NEXT: vmovdqa %xmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[3,1,2,3]
-; AVX2-FCP-NEXT: vmovdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,1,2,0,4,5,6,7]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm6 = xmm6[0,1,2,0,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
-; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm13 = xmm3[3,1,2,3]
-; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm12 = xmm5[3,1,2,3]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm13[2,0,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm0 = xmm9[3,1,2,3]
+; AVX2-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm3 = xmm8[3,1,2,3]
+; AVX2-FCP-NEXT: vmovdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[0,1,2,0,4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,2,0,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm0 = xmm5[3,1,2,3]
+; AVX2-FCP-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; AVX2-FCP-NEXT: vpshufd {{.*#+}} xmm12 = xmm10[3,1,2,3]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm0[2,0,2,3,4,5,6,7]
; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[2,0,2,3,4,5,6,7]
; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-FCP-NEXT: vmovdqu %ymm1, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
-; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm8 # 32-byte Folded Reload
-; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm0, %ymm7 # 32-byte Folded Reload
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm8, %ymm0
-; AVX2-FCP-NEXT: vpshufb %ymm4, %ymm7, %ymm1
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm3 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-FCP-NEXT: vpshufd $231, (%rsp), %xmm6 # 16-byte Folded Reload
-; AVX2-FCP-NEXT: # xmm6 = mem[3,1,2,3]
-; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm5 # 16-byte Folded Reload
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
+; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm8 # 32-byte Folded Reload
+; AVX2-FCP-NEXT: vpermd {{[-0-9]+}}(%r{{[sb]}}p), %ymm6, %ymm9 # 32-byte Folded Reload
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm8, %ymm0
+; AVX2-FCP-NEXT: vpshufb %ymm14, %ymm9, %ymm1
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm1[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-FCP-NEXT: vpshufd $231, (%rsp), %xmm5 # 16-byte Folded Reload
; AVX2-FCP-NEXT: # xmm5 = mem[3,1,2,3]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm6[0,1,2,0,4,5,6,7]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm5[0,1,2,0,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
-; AVX2-FCP-NEXT: # xmm4 = mem[3,1,2,3]
+; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
+; AVX2-FCP-NEXT: # xmm3 = mem[3,1,2,3]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm5[0,1,2,0,4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm3[0,1,2,0,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
; AVX2-FCP-NEXT: # xmm2 = mem[3,1,2,3]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm4[2,0,2,3,4,5,6,7]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm15 = xmm2[2,0,2,3,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm15[0],xmm0[0],xmm15[1],xmm0[1]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm3[4,5,6,7]
+; AVX2-FCP-NEXT: vpshufd $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
+; AVX2-FCP-NEXT: # xmm1 = mem[3,1,2,3]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm0 = xmm2[2,0,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm14 = xmm1[2,0,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm14[0],xmm0[0],xmm14[1],xmm0[1]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm10[2,3]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
; AVX2-FCP-NEXT: vmovdqu %ymm0, {{[-0-9]+}}(%r{{[sb]}}p) # 32-byte Spill
; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm0, %ymm0
-; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm1, %ymm1
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX2-FCP-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
-; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
-; AVX2-FCP-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; AVX2-FCP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
-; AVX2-FCP-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
-; AVX2-FCP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm15 # 16-byte Folded Reload
-; AVX2-FCP-NEXT: # xmm15 = mem[3,1,2,3,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm15[0],xmm3[0],xmm15[1],xmm3[1]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm15 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm0, %ymm0
+; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm4 # 32-byte Reload
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm4, %ymm4
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm4[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
+; AVX2-FCP-NEXT: # xmm4 = mem[0,1,3,1,4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
+; AVX2-FCP-NEXT: # xmm10 = mem[0,1,3,1,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm10[0],xmm4[0],xmm10[1],xmm4[1]
+; AVX2-FCP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
+; AVX2-FCP-NEXT: # xmm10 = mem[3,1,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm14 # 16-byte Folded Reload
+; AVX2-FCP-NEXT: # xmm14 = mem[3,1,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm14[0],xmm10[0],xmm14[1],xmm10[1]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm4 = xmm10[0,1],xmm4[2,3]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm4 = ymm4[0,1,2,3],ymm0[4,5,6,7]
; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm0 # 32-byte Reload
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm0, %ymm0
-; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm1, %ymm1
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3,4,5],ymm0[6,7]
-; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Folded Reload
-; AVX2-FCP-NEXT: # xmm1 = mem[0,1,3,1,4,5,6,7]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm14[0,1,3,1,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm11[3,1,2,3,4,5,6,7]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm9 = xmm9[3,1,2,3,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm9[0],xmm3[0],xmm9[1],xmm3[1]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm3[0,1],xmm1[2,3]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm8, %ymm1
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm7, %ymm3
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm6[0,1,3,1,4,5,6,7]
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm0, %ymm0
+; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm6 # 32-byte Reload
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm6, %ymm10
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm10[0,1,2,3,4,5],ymm0[6,7]
+; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm10 # 16-byte Folded Reload
+; AVX2-FCP-NEXT: # xmm10 = mem[0,1,3,1,4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm11 = xmm11[0,1,3,1,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm10 = xmm11[0],xmm10[0],xmm11[1],xmm10[1]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm7 = xmm7[3,1,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm11 = xmm15[3,1,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm7 = xmm11[0],xmm7[0],xmm11[1],xmm7[1]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm7 = xmm7[0,1],xmm10[2,3]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm7 = ymm7[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm8, %ymm0
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm9, %ymm6
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm6[0,1,2,3,4,5],ymm0[6,7]
; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm5[0,1,3,1,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm4[3,1,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm3 = xmm3[0,1,3,1,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[3,1,2,3,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0,1],xmm3[2,3]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm1 = xmm1[3,1,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm1 # 32-byte Reload
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm1, %ymm1
; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm2, %ymm2
-; AVX2-FCP-NEXT: vmovdqu {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vpshufb %ymm10, %ymm3, %ymm3
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3,4,5],ymm2[6,7]
+; AVX2-FCP-NEXT: vpshufb %ymm13, %ymm2, %ymm2
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm2 # 16-byte Folded Reload
+; AVX2-FCP-NEXT: # xmm2 = mem[0,1,3,1,4,5,6,7]
; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
; AVX2-FCP-NEXT: # xmm3 = mem[0,1,3,1,4,5,6,7]
-; AVX2-FCP-NEXT: vpshuflw $116, {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Folded Reload
-; AVX2-FCP-NEXT: # xmm4 = mem[0,1,3,1,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm4 = xmm13[3,1,2,3,4,5,6,7]
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
+; AVX2-FCP-NEXT: vpshuflw $231, {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Folded Reload
+; AVX2-FCP-NEXT: # xmm3 = mem[3,1,2,3,4,5,6,7]
; AVX2-FCP-NEXT: vpshuflw {{.*#+}} xmm5 = xmm12[3,1,2,3,4,5,6,7]
-; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm3 = xmm4[0,1],xmm3[2,3]
-; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vmovaps %ymm3, 32(%rsi)
-; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vmovaps %ymm3, 96(%rsi)
-; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vmovaps %ymm3, 64(%rsi)
-; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vmovaps %ymm3, (%rsi)
-; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vmovaps %ymm3, 96(%rdx)
-; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vmovaps %ymm3, 32(%rdx)
-; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vmovaps %ymm3, (%rdx)
-; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vmovaps %ymm3, 64(%rdx)
-; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vmovaps %ymm3, 32(%rcx)
-; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vmovaps %ymm3, 96(%rcx)
-; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vmovaps %ymm3, 64(%rcx)
-; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm3 # 32-byte Reload
-; AVX2-FCP-NEXT: vmovaps %ymm3, (%rcx)
-; AVX2-FCP-NEXT: vmovdqa %ymm2, 96(%r8)
-; AVX2-FCP-NEXT: vmovdqa %ymm1, 32(%r8)
-; AVX2-FCP-NEXT: vmovdqa %ymm0, 64(%r8)
-; AVX2-FCP-NEXT: vmovdqa %ymm15, (%r8)
+; AVX2-FCP-NEXT: vpunpckldq {{.*#+}} xmm3 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} xmm2 = xmm3[0,1],xmm2[2,3]
+; AVX2-FCP-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rsi)
+; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FCP-NEXT: vmovaps %ymm2, 96(%rsi)
+; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FCP-NEXT: vmovaps %ymm2, 64(%rsi)
+; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FCP-NEXT: vmovaps %ymm2, (%rsi)
+; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FCP-NEXT: vmovaps %ymm2, 96(%rdx)
+; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rdx)
+; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FCP-NEXT: vmovaps %ymm2, (%rdx)
+; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FCP-NEXT: vmovaps %ymm2, 64(%rdx)
+; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FCP-NEXT: vmovaps %ymm2, 32(%rcx)
+; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FCP-NEXT: vmovaps %ymm2, 96(%rcx)
+; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FCP-NEXT: vmovaps %ymm2, 64(%rcx)
+; AVX2-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %ymm2 # 32-byte Reload
+; AVX2-FCP-NEXT: vmovaps %ymm2, (%rcx)
+; AVX2-FCP-NEXT: vmovdqa %ymm1, 96(%r8)
+; AVX2-FCP-NEXT: vmovdqa %ymm0, 32(%r8)
+; AVX2-FCP-NEXT: vmovdqa %ymm7, 64(%r8)
+; AVX2-FCP-NEXT: vmovdqa %ymm4, (%r8)
; AVX2-FCP-NEXT: addq $680, %rsp # imm = 0x2A8
; AVX2-FCP-NEXT: vzeroupper
; AVX2-FCP-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/vector-lzcnt-256.ll b/llvm/test/CodeGen/X86/vector-lzcnt-256.ll
index db363493e2dac..39b28fd0477b2 100644
--- a/llvm/test/CodeGen/X86/vector-lzcnt-256.ll
+++ b/llvm/test/CodeGen/X86/vector-lzcnt-256.ll
@@ -68,24 +68,24 @@ define <4 x i64> @testv4i64(<4 x i64> %in) nounwind {
; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX2-NEXT: # ymm1 = mem[0,1,0,1]
; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
-; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; AVX2-NEXT: vpsrlw $8, %ymm2, %ymm2
; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX2-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2
+; AVX2-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm2
; AVX2-NEXT: vpsrld $16, %ymm2, %ymm2
; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX2-NEXT: vpsrld $16, %ymm1, %ymm1
; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpcmpeqd %ymm3, %ymm0, %ymm0
; AVX2-NEXT: vpsrlq $32, %ymm0, %ymm0
; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vpsrlq $32, %ymm1, %ymm1
@@ -97,24 +97,24 @@ define <4 x i64> @testv4i64(<4 x i64> %in) nounwind {
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3
-; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX512VL-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; AVX512VL-NEXT: vpsrlw $8, %ymm2, %ymm2
; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2
+; AVX512VL-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm2
; AVX512VL-NEXT: vpsrld $16, %ymm2, %ymm2
; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX512VL-NEXT: vpsrld $16, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddd %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT: vpcmpeqd %ymm3, %ymm0, %ymm0
; AVX512VL-NEXT: vpsrlq $32, %ymm0, %ymm0
; AVX512VL-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512VL-NEXT: vpsrlq $32, %ymm1, %ymm1
@@ -126,24 +126,24 @@ define <4 x i64> @testv4i64(<4 x i64> %in) nounwind {
; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3
-; AVX512VLBWDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX512VLBWDQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX512VLBWDQ-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX512VLBWDQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512VLBWDQ-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512VLBWDQ-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm2, %ymm2
; AVX512VLBWDQ-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512VLBWDQ-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; AVX512VLBWDQ-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2
+; AVX512VLBWDQ-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm2
; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm2, %ymm2
; AVX512VLBWDQ-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm1, %ymm1
; AVX512VLBWDQ-NEXT: vpaddd %ymm2, %ymm1, %ymm1
-; AVX512VLBWDQ-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0
+; AVX512VLBWDQ-NEXT: vpcmpeqd %ymm3, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpsrlq $32, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512VLBWDQ-NEXT: vpsrlq $32, %ymm1, %ymm1
@@ -164,30 +164,30 @@ define <4 x i64> @testv4i64(<4 x i64> %in) nounwind {
;
; X86-AVX-LABEL: testv4i64:
; X86-AVX: # %bb.0:
-; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]
-; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm3
-; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3
-; X86-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
-; X86-AVX-NEXT: vpand %ymm5, %ymm2, %ymm2
-; X86-AVX-NEXT: vpshufb %ymm3, %ymm1, %ymm1
-; X86-AVX-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; X86-AVX-NEXT: # ymm2 = mem[0,1,0,1]
+; X86-AVX-NEXT: vpshufb %ymm0, %ymm2, %ymm3
+; X86-AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm4
+; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm4, %ymm4
+; X86-AVX-NEXT: vpcmpeqb %ymm1, %ymm4, %ymm5
+; X86-AVX-NEXT: vpand %ymm5, %ymm3, %ymm3
+; X86-AVX-NEXT: vpshufb %ymm4, %ymm2, %ymm2
+; X86-AVX-NEXT: vpaddb %ymm2, %ymm3, %ymm2
+; X86-AVX-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm3
+; X86-AVX-NEXT: vpsrlw $8, %ymm3, %ymm3
+; X86-AVX-NEXT: vpand %ymm3, %ymm2, %ymm3
; X86-AVX-NEXT: vpsrlw $8, %ymm2, %ymm2
-; X86-AVX-NEXT: vpand %ymm2, %ymm1, %ymm2
-; X86-AVX-NEXT: vpsrlw $8, %ymm1, %ymm1
-; X86-AVX-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; X86-AVX-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2
+; X86-AVX-NEXT: vpaddw %ymm3, %ymm2, %ymm2
+; X86-AVX-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm3
+; X86-AVX-NEXT: vpsrld $16, %ymm3, %ymm3
+; X86-AVX-NEXT: vpand %ymm3, %ymm2, %ymm3
; X86-AVX-NEXT: vpsrld $16, %ymm2, %ymm2
-; X86-AVX-NEXT: vpand %ymm2, %ymm1, %ymm2
-; X86-AVX-NEXT: vpsrld $16, %ymm1, %ymm1
-; X86-AVX-NEXT: vpaddd %ymm2, %ymm1, %ymm1
-; X86-AVX-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0
+; X86-AVX-NEXT: vpaddd %ymm3, %ymm2, %ymm2
+; X86-AVX-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm0
; X86-AVX-NEXT: vpsrlq $32, %ymm0, %ymm0
-; X86-AVX-NEXT: vpand %ymm0, %ymm1, %ymm0
-; X86-AVX-NEXT: vpsrlq $32, %ymm1, %ymm1
+; X86-AVX-NEXT: vpand %ymm0, %ymm2, %ymm0
+; X86-AVX-NEXT: vpsrlq $32, %ymm2, %ymm1
; X86-AVX-NEXT: vpaddq %ymm0, %ymm1, %ymm0
; X86-AVX-NEXT: retl
@@ -254,24 +254,24 @@ define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind {
; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX2-NEXT: # ymm1 = mem[0,1,0,1]
; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
-; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; AVX2-NEXT: vpsrlw $8, %ymm2, %ymm2
; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX2-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2
+; AVX2-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm2
; AVX2-NEXT: vpsrld $16, %ymm2, %ymm2
; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX2-NEXT: vpsrld $16, %ymm1, %ymm1
; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpcmpeqd %ymm3, %ymm0, %ymm0
; AVX2-NEXT: vpsrlq $32, %ymm0, %ymm0
; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vpsrlq $32, %ymm1, %ymm1
@@ -283,24 +283,24 @@ define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind {
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3
-; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX512VL-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; AVX512VL-NEXT: vpsrlw $8, %ymm2, %ymm2
; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2
+; AVX512VL-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm2
; AVX512VL-NEXT: vpsrld $16, %ymm2, %ymm2
; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX512VL-NEXT: vpsrld $16, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddd %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT: vpcmpeqd %ymm3, %ymm0, %ymm0
; AVX512VL-NEXT: vpsrlq $32, %ymm0, %ymm0
; AVX512VL-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512VL-NEXT: vpsrlq $32, %ymm1, %ymm1
@@ -312,24 +312,24 @@ define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind {
; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3
-; AVX512VLBWDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX512VLBWDQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX512VLBWDQ-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX512VLBWDQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512VLBWDQ-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512VLBWDQ-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm2, %ymm2
; AVX512VLBWDQ-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512VLBWDQ-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; AVX512VLBWDQ-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2
+; AVX512VLBWDQ-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm2
; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm2, %ymm2
; AVX512VLBWDQ-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm1, %ymm1
; AVX512VLBWDQ-NEXT: vpaddd %ymm2, %ymm1, %ymm1
-; AVX512VLBWDQ-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0
+; AVX512VLBWDQ-NEXT: vpcmpeqd %ymm3, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpsrlq $32, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512VLBWDQ-NEXT: vpsrlq $32, %ymm1, %ymm1
@@ -350,30 +350,30 @@ define <4 x i64> @testv4i64u(<4 x i64> %in) nounwind {
;
; X86-AVX-LABEL: testv4i64u:
; X86-AVX: # %bb.0:
-; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
-; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]
-; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm3
-; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3
-; X86-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
-; X86-AVX-NEXT: vpand %ymm5, %ymm2, %ymm2
-; X86-AVX-NEXT: vpshufb %ymm3, %ymm1, %ymm1
-; X86-AVX-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm2 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
+; X86-AVX-NEXT: # ymm2 = mem[0,1,0,1]
+; X86-AVX-NEXT: vpshufb %ymm0, %ymm2, %ymm3
+; X86-AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm4
+; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm4, %ymm4
+; X86-AVX-NEXT: vpcmpeqb %ymm1, %ymm4, %ymm5
+; X86-AVX-NEXT: vpand %ymm5, %ymm3, %ymm3
+; X86-AVX-NEXT: vpshufb %ymm4, %ymm2, %ymm2
+; X86-AVX-NEXT: vpaddb %ymm2, %ymm3, %ymm2
+; X86-AVX-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm3
+; X86-AVX-NEXT: vpsrlw $8, %ymm3, %ymm3
+; X86-AVX-NEXT: vpand %ymm3, %ymm2, %ymm3
; X86-AVX-NEXT: vpsrlw $8, %ymm2, %ymm2
-; X86-AVX-NEXT: vpand %ymm2, %ymm1, %ymm2
-; X86-AVX-NEXT: vpsrlw $8, %ymm1, %ymm1
-; X86-AVX-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; X86-AVX-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm2
+; X86-AVX-NEXT: vpaddw %ymm3, %ymm2, %ymm2
+; X86-AVX-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm3
+; X86-AVX-NEXT: vpsrld $16, %ymm3, %ymm3
+; X86-AVX-NEXT: vpand %ymm3, %ymm2, %ymm3
; X86-AVX-NEXT: vpsrld $16, %ymm2, %ymm2
-; X86-AVX-NEXT: vpand %ymm2, %ymm1, %ymm2
-; X86-AVX-NEXT: vpsrld $16, %ymm1, %ymm1
-; X86-AVX-NEXT: vpaddd %ymm2, %ymm1, %ymm1
-; X86-AVX-NEXT: vpcmpeqd %ymm4, %ymm0, %ymm0
+; X86-AVX-NEXT: vpaddd %ymm3, %ymm2, %ymm2
+; X86-AVX-NEXT: vpcmpeqd %ymm1, %ymm0, %ymm0
; X86-AVX-NEXT: vpsrlq $32, %ymm0, %ymm0
-; X86-AVX-NEXT: vpand %ymm0, %ymm1, %ymm0
-; X86-AVX-NEXT: vpsrlq $32, %ymm1, %ymm1
+; X86-AVX-NEXT: vpand %ymm0, %ymm2, %ymm0
+; X86-AVX-NEXT: vpsrlq $32, %ymm2, %ymm1
; X86-AVX-NEXT: vpaddq %ymm0, %ymm1, %ymm0
; X86-AVX-NEXT: retl
@@ -430,19 +430,19 @@ define <8 x i32> @testv8i32(<8 x i32> %in) nounwind {
; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX2-NEXT: # ymm1 = mem[0,1,0,1]
; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
-; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; AVX2-NEXT: vpsrlw $8, %ymm2, %ymm2
; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX2-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm0
; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0
; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vpsrld $16, %ymm1, %ymm1
@@ -454,19 +454,19 @@ define <8 x i32> @testv8i32(<8 x i32> %in) nounwind {
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3
-; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX512VL-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; AVX512VL-NEXT: vpsrlw $8, %ymm2, %ymm2
; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm0
; AVX512VL-NEXT: vpsrld $16, %ymm0, %ymm0
; AVX512VL-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512VL-NEXT: vpsrld $16, %ymm1, %ymm1
@@ -478,19 +478,19 @@ define <8 x i32> @testv8i32(<8 x i32> %in) nounwind {
; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3
-; AVX512VLBWDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX512VLBWDQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX512VLBWDQ-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX512VLBWDQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512VLBWDQ-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512VLBWDQ-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm2, %ymm2
; AVX512VLBWDQ-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512VLBWDQ-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; AVX512VLBWDQ-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0
+; AVX512VLBWDQ-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm1, %ymm1
@@ -514,19 +514,19 @@ define <8 x i32> @testv8i32(<8 x i32> %in) nounwind {
; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]
; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm3
-; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3
-; X86-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; X86-AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm4
+; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm4, %ymm4
+; X86-AVX-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; X86-AVX-NEXT: vpand %ymm5, %ymm2, %ymm2
-; X86-AVX-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; X86-AVX-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; X86-AVX-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; X86-AVX-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; X86-AVX-NEXT: vpsrlw $8, %ymm2, %ymm2
; X86-AVX-NEXT: vpand %ymm2, %ymm1, %ymm2
; X86-AVX-NEXT: vpsrlw $8, %ymm1, %ymm1
; X86-AVX-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; X86-AVX-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0
+; X86-AVX-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm0
; X86-AVX-NEXT: vpsrld $16, %ymm0, %ymm0
; X86-AVX-NEXT: vpand %ymm0, %ymm1, %ymm0
; X86-AVX-NEXT: vpsrld $16, %ymm1, %ymm1
@@ -586,19 +586,19 @@ define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind {
; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX2-NEXT: # ymm1 = mem[0,1,0,1]
; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
-; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; AVX2-NEXT: vpsrlw $8, %ymm2, %ymm2
; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX2-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm0
; AVX2-NEXT: vpsrld $16, %ymm0, %ymm0
; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vpsrld $16, %ymm1, %ymm1
@@ -610,19 +610,19 @@ define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind {
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3
-; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX512VL-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; AVX512VL-NEXT: vpsrlw $8, %ymm2, %ymm2
; AVX512VL-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm0
; AVX512VL-NEXT: vpsrld $16, %ymm0, %ymm0
; AVX512VL-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512VL-NEXT: vpsrld $16, %ymm1, %ymm1
@@ -634,19 +634,19 @@ define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind {
; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3
-; AVX512VLBWDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX512VLBWDQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX512VLBWDQ-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX512VLBWDQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512VLBWDQ-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512VLBWDQ-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm2, %ymm2
; AVX512VLBWDQ-NEXT: vpand %ymm2, %ymm1, %ymm2
; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512VLBWDQ-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; AVX512VLBWDQ-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0
+; AVX512VLBWDQ-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512VLBWDQ-NEXT: vpsrld $16, %ymm1, %ymm1
@@ -670,19 +670,19 @@ define <8 x i32> @testv8i32u(<8 x i32> %in) nounwind {
; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]
; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm3
-; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3
-; X86-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; X86-AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm4
+; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm4, %ymm4
+; X86-AVX-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; X86-AVX-NEXT: vpand %ymm5, %ymm2, %ymm2
-; X86-AVX-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; X86-AVX-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; X86-AVX-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm2
+; X86-AVX-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm2
; X86-AVX-NEXT: vpsrlw $8, %ymm2, %ymm2
; X86-AVX-NEXT: vpand %ymm2, %ymm1, %ymm2
; X86-AVX-NEXT: vpsrlw $8, %ymm1, %ymm1
; X86-AVX-NEXT: vpaddw %ymm2, %ymm1, %ymm1
-; X86-AVX-NEXT: vpcmpeqw %ymm4, %ymm0, %ymm0
+; X86-AVX-NEXT: vpcmpeqw %ymm3, %ymm0, %ymm0
; X86-AVX-NEXT: vpsrld $16, %ymm0, %ymm0
; X86-AVX-NEXT: vpand %ymm0, %ymm1, %ymm0
; X86-AVX-NEXT: vpsrld $16, %ymm1, %ymm1
@@ -732,14 +732,14 @@ define <16 x i16> @testv16i16(<16 x i16> %in) nounwind {
; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX2-NEXT: # ymm1 = mem[0,1,0,1]
; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
-; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm0
; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
@@ -751,14 +751,14 @@ define <16 x i16> @testv16i16(<16 x i16> %in) nounwind {
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3
-; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX512VL-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm0
; AVX512VL-NEXT: vpsrlw $8, %ymm0, %ymm0
; AVX512VL-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1
@@ -770,14 +770,14 @@ define <16 x i16> @testv16i16(<16 x i16> %in) nounwind {
; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3
-; AVX512VLBWDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX512VLBWDQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX512VLBWDQ-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX512VLBWDQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512VLBWDQ-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512VLBWDQ-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0
+; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm1, %ymm1
@@ -797,14 +797,14 @@ define <16 x i16> @testv16i16(<16 x i16> %in) nounwind {
; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]
; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm3
-; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3
-; X86-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; X86-AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm4
+; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm4, %ymm4
+; X86-AVX-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; X86-AVX-NEXT: vpand %ymm5, %ymm2, %ymm2
-; X86-AVX-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; X86-AVX-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; X86-AVX-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0
+; X86-AVX-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm0
; X86-AVX-NEXT: vpsrlw $8, %ymm0, %ymm0
; X86-AVX-NEXT: vpand %ymm0, %ymm1, %ymm0
; X86-AVX-NEXT: vpsrlw $8, %ymm1, %ymm1
@@ -853,14 +853,14 @@ define <16 x i16> @testv16i16u(<16 x i16> %in) nounwind {
; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX2-NEXT: # ymm1 = mem[0,1,0,1]
; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm3, %ymm3
-; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm4, %ymm4
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX2-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX2-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm0
; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
; AVX2-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
@@ -872,14 +872,14 @@ define <16 x i16> @testv16i16u(<16 x i16> %in) nounwind {
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3
-; AVX512VL-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX512VL-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX512VL-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512VL-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512VL-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX512VL-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0
+; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm0
; AVX512VL-NEXT: vpsrlw $8, %ymm0, %ymm0
; AVX512VL-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512VL-NEXT: vpsrlw $8, %ymm1, %ymm1
@@ -891,14 +891,14 @@ define <16 x i16> @testv16i16u(<16 x i16> %in) nounwind {
; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm3
-; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm3, %ymm3
-; AVX512VLBWDQ-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; AVX512VLBWDQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm4
+; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm4, %ymm4
+; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; AVX512VLBWDQ-NEXT: vpand %ymm5, %ymm2, %ymm2
-; AVX512VLBWDQ-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; AVX512VLBWDQ-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; AVX512VLBWDQ-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0
+; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpand %ymm0, %ymm1, %ymm0
; AVX512VLBWDQ-NEXT: vpsrlw $8, %ymm1, %ymm1
@@ -918,14 +918,14 @@ define <16 x i16> @testv16i16u(<16 x i16> %in) nounwind {
; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]
; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2
-; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm3
-; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm3, %ymm3
-; X86-AVX-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm3, %ymm5
+; X86-AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm4
+; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm4, %ymm4
+; X86-AVX-NEXT: vpcmpeqb %ymm3, %ymm4, %ymm5
; X86-AVX-NEXT: vpand %ymm5, %ymm2, %ymm2
-; X86-AVX-NEXT: vpshufb %ymm3, %ymm1, %ymm1
+; X86-AVX-NEXT: vpshufb %ymm4, %ymm1, %ymm1
; X86-AVX-NEXT: vpaddb %ymm1, %ymm2, %ymm1
-; X86-AVX-NEXT: vpcmpeqb %ymm4, %ymm0, %ymm0
+; X86-AVX-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm0
; X86-AVX-NEXT: vpsrlw $8, %ymm0, %ymm0
; X86-AVX-NEXT: vpand %ymm0, %ymm1, %ymm0
; X86-AVX-NEXT: vpsrlw $8, %ymm1, %ymm1
@@ -964,9 +964,9 @@ define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX2-NEXT: # ymm1 = mem[0,1,0,1]
; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm3
; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm0
@@ -978,9 +978,9 @@ define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2
+; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0
; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm3
; AVX512VL-NEXT: vpand %ymm3, %ymm2, %ymm2
; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm0
@@ -992,9 +992,9 @@ define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2
+; AVX512VLBWDQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VLBWDQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm3
; AVX512VLBWDQ-NEXT: vpand %ymm3, %ymm2, %ymm2
; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm0
@@ -1019,9 +1019,9 @@ define <32 x i8> @testv32i8(<32 x i8> %in) nounwind {
; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]
; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2
+; X86-AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3
; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm0
; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
-; X86-AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3
; X86-AVX-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm3
; X86-AVX-NEXT: vpand %ymm3, %ymm2, %ymm2
; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm0
@@ -1060,9 +1060,9 @@ define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX2-NEXT: # ymm1 = mem[0,1,0,1]
; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm2
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm3
; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
; AVX2-NEXT: vpshufb %ymm0, %ymm1, %ymm0
@@ -1074,9 +1074,9 @@ define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
; AVX512VL-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VL-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm2
+; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0
; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512VL-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm3
; AVX512VL-NEXT: vpand %ymm3, %ymm2, %ymm2
; AVX512VL-NEXT: vpshufb %ymm0, %ymm1, %ymm0
@@ -1088,9 +1088,9 @@ define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
; AVX512VLBWDQ-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; AVX512VLBWDQ-NEXT: # ymm1 = mem[0,1,0,1]
; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm2
+; AVX512VLBWDQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512VLBWDQ-NEXT: vpsrlw $4, %ymm0, %ymm0
; AVX512VLBWDQ-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm0, %ymm0
-; AVX512VLBWDQ-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512VLBWDQ-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm3
; AVX512VLBWDQ-NEXT: vpand %ymm3, %ymm2, %ymm2
; AVX512VLBWDQ-NEXT: vpshufb %ymm0, %ymm1, %ymm0
@@ -1115,9 +1115,9 @@ define <32 x i8> @testv32i8u(<32 x i8> %in) nounwind {
; X86-AVX-NEXT: vbroadcasti128 {{.*#+}} ymm1 = [4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0,4,3,2,2,1,1,1,1,0,0,0,0,0,0,0,0]
; X86-AVX-NEXT: # ymm1 = mem[0,1,0,1]
; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm2
+; X86-AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3
; X86-AVX-NEXT: vpsrlw $4, %ymm0, %ymm0
; X86-AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
-; X86-AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3
; X86-AVX-NEXT: vpcmpeqb %ymm3, %ymm0, %ymm3
; X86-AVX-NEXT: vpand %ymm3, %ymm2, %ymm2
; X86-AVX-NEXT: vpshufb %ymm0, %ymm1, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-popcnt-256-ult-ugt.ll b/llvm/test/CodeGen/X86/vector-popcnt-256-ult-ugt.ll
index 487f9a5d326cf..6167bc2a21fbf 100644
--- a/llvm/test/CodeGen/X86/vector-popcnt-256-ult-ugt.ll
+++ b/llvm/test/CodeGen/X86/vector-popcnt-256-ult-ugt.ll
@@ -3516,8 +3516,8 @@ define <8 x i32> @ugt_2_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_2_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -3618,8 +3618,8 @@ define <8 x i32> @ult_3_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_3_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -3721,8 +3721,8 @@ define <8 x i32> @ugt_3_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_3_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -3823,8 +3823,8 @@ define <8 x i32> @ult_4_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_4_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -3926,8 +3926,8 @@ define <8 x i32> @ugt_4_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_4_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -4028,8 +4028,8 @@ define <8 x i32> @ult_5_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_5_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -4131,8 +4131,8 @@ define <8 x i32> @ugt_5_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_5_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -4233,8 +4233,8 @@ define <8 x i32> @ult_6_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_6_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -4336,8 +4336,8 @@ define <8 x i32> @ugt_6_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_6_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -4438,8 +4438,8 @@ define <8 x i32> @ult_7_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_7_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -4541,8 +4541,8 @@ define <8 x i32> @ugt_7_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_7_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -4643,8 +4643,8 @@ define <8 x i32> @ult_8_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_8_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -4746,8 +4746,8 @@ define <8 x i32> @ugt_8_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_8_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -4848,8 +4848,8 @@ define <8 x i32> @ult_9_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_9_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -4951,8 +4951,8 @@ define <8 x i32> @ugt_9_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_9_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -5053,8 +5053,8 @@ define <8 x i32> @ult_10_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_10_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -5156,8 +5156,8 @@ define <8 x i32> @ugt_10_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_10_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -5258,8 +5258,8 @@ define <8 x i32> @ult_11_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_11_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -5361,8 +5361,8 @@ define <8 x i32> @ugt_11_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_11_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -5463,8 +5463,8 @@ define <8 x i32> @ult_12_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_12_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -5566,8 +5566,8 @@ define <8 x i32> @ugt_12_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_12_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -5668,8 +5668,8 @@ define <8 x i32> @ult_13_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_13_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -5771,8 +5771,8 @@ define <8 x i32> @ugt_13_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_13_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -5873,8 +5873,8 @@ define <8 x i32> @ult_14_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_14_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -5976,8 +5976,8 @@ define <8 x i32> @ugt_14_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_14_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -6078,8 +6078,8 @@ define <8 x i32> @ult_15_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_15_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -6181,8 +6181,8 @@ define <8 x i32> @ugt_15_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_15_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -6283,8 +6283,8 @@ define <8 x i32> @ult_16_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_16_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -6386,8 +6386,8 @@ define <8 x i32> @ugt_16_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_16_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -6488,8 +6488,8 @@ define <8 x i32> @ult_17_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_17_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -6591,8 +6591,8 @@ define <8 x i32> @ugt_17_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_17_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -6693,8 +6693,8 @@ define <8 x i32> @ult_18_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_18_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -6796,8 +6796,8 @@ define <8 x i32> @ugt_18_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_18_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -6898,8 +6898,8 @@ define <8 x i32> @ult_19_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_19_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -7001,8 +7001,8 @@ define <8 x i32> @ugt_19_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_19_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -7103,8 +7103,8 @@ define <8 x i32> @ult_20_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_20_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -7206,8 +7206,8 @@ define <8 x i32> @ugt_20_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_20_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -7308,8 +7308,8 @@ define <8 x i32> @ult_21_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_21_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -7411,8 +7411,8 @@ define <8 x i32> @ugt_21_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_21_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -7513,8 +7513,8 @@ define <8 x i32> @ult_22_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_22_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -7616,8 +7616,8 @@ define <8 x i32> @ugt_22_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_22_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -7718,8 +7718,8 @@ define <8 x i32> @ult_23_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_23_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -7821,8 +7821,8 @@ define <8 x i32> @ugt_23_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_23_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -7923,8 +7923,8 @@ define <8 x i32> @ult_24_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_24_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -8026,8 +8026,8 @@ define <8 x i32> @ugt_24_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_24_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -8128,8 +8128,8 @@ define <8 x i32> @ult_25_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_25_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -8231,8 +8231,8 @@ define <8 x i32> @ugt_25_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_25_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -8333,8 +8333,8 @@ define <8 x i32> @ult_26_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_26_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -8436,8 +8436,8 @@ define <8 x i32> @ugt_26_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_26_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -8538,8 +8538,8 @@ define <8 x i32> @ult_27_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_27_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -8641,8 +8641,8 @@ define <8 x i32> @ugt_27_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_27_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -8743,8 +8743,8 @@ define <8 x i32> @ult_28_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_28_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -8846,8 +8846,8 @@ define <8 x i32> @ugt_28_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_28_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -8948,8 +8948,8 @@ define <8 x i32> @ult_29_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_29_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -9051,8 +9051,8 @@ define <8 x i32> @ugt_29_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_29_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -9153,8 +9153,8 @@ define <8 x i32> @ult_30_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_30_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -9256,8 +9256,8 @@ define <8 x i32> @ugt_30_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ugt_30_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -9358,8 +9358,8 @@ define <8 x i32> @ult_31_v8i32(<8 x i32> %0) {
;
; BITALG-LABEL: ult_31_v8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
@@ -9575,8 +9575,8 @@ define <4 x i64> @ugt_2_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_2_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [2,2,2,2]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -9658,8 +9658,8 @@ define <4 x i64> @ult_3_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_3_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [3,3,3,3]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -9741,8 +9741,8 @@ define <4 x i64> @ugt_3_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_3_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [3,3,3,3]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -9824,8 +9824,8 @@ define <4 x i64> @ult_4_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_4_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [4,4,4,4]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -9907,8 +9907,8 @@ define <4 x i64> @ugt_4_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_4_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [4,4,4,4]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -9990,8 +9990,8 @@ define <4 x i64> @ult_5_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_5_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [5,5,5,5]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -10073,8 +10073,8 @@ define <4 x i64> @ugt_5_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_5_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [5,5,5,5]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -10156,8 +10156,8 @@ define <4 x i64> @ult_6_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_6_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [6,6,6,6]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -10239,8 +10239,8 @@ define <4 x i64> @ugt_6_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_6_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [6,6,6,6]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -10322,8 +10322,8 @@ define <4 x i64> @ult_7_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_7_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [7,7,7,7]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -10405,8 +10405,8 @@ define <4 x i64> @ugt_7_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_7_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [7,7,7,7]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -10488,8 +10488,8 @@ define <4 x i64> @ult_8_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_8_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [8,8,8,8]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -10571,8 +10571,8 @@ define <4 x i64> @ugt_8_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_8_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [8,8,8,8]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -10654,8 +10654,8 @@ define <4 x i64> @ult_9_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_9_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [9,9,9,9]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -10737,8 +10737,8 @@ define <4 x i64> @ugt_9_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_9_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [9,9,9,9]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -10820,8 +10820,8 @@ define <4 x i64> @ult_10_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_10_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [10,10,10,10]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -10903,8 +10903,8 @@ define <4 x i64> @ugt_10_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_10_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [10,10,10,10]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -10986,8 +10986,8 @@ define <4 x i64> @ult_11_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_11_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [11,11,11,11]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -11069,8 +11069,8 @@ define <4 x i64> @ugt_11_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_11_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [11,11,11,11]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -11152,8 +11152,8 @@ define <4 x i64> @ult_12_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_12_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [12,12,12,12]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -11235,8 +11235,8 @@ define <4 x i64> @ugt_12_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_12_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [12,12,12,12]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -11318,8 +11318,8 @@ define <4 x i64> @ult_13_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_13_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [13,13,13,13]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -11401,8 +11401,8 @@ define <4 x i64> @ugt_13_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_13_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [13,13,13,13]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -11484,8 +11484,8 @@ define <4 x i64> @ult_14_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_14_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [14,14,14,14]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -11567,8 +11567,8 @@ define <4 x i64> @ugt_14_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_14_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [14,14,14,14]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -11650,8 +11650,8 @@ define <4 x i64> @ult_15_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_15_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [15,15,15,15]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -11733,8 +11733,8 @@ define <4 x i64> @ugt_15_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_15_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [15,15,15,15]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -11816,8 +11816,8 @@ define <4 x i64> @ult_16_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_16_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [16,16,16,16]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -11899,8 +11899,8 @@ define <4 x i64> @ugt_16_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_16_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [16,16,16,16]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -11982,8 +11982,8 @@ define <4 x i64> @ult_17_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_17_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [17,17,17,17]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -12065,8 +12065,8 @@ define <4 x i64> @ugt_17_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_17_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [17,17,17,17]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -12148,8 +12148,8 @@ define <4 x i64> @ult_18_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_18_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [18,18,18,18]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -12231,8 +12231,8 @@ define <4 x i64> @ugt_18_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_18_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [18,18,18,18]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -12314,8 +12314,8 @@ define <4 x i64> @ult_19_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_19_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [19,19,19,19]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -12397,8 +12397,8 @@ define <4 x i64> @ugt_19_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_19_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [19,19,19,19]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -12480,8 +12480,8 @@ define <4 x i64> @ult_20_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_20_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [20,20,20,20]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -12563,8 +12563,8 @@ define <4 x i64> @ugt_20_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_20_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [20,20,20,20]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -12646,8 +12646,8 @@ define <4 x i64> @ult_21_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_21_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [21,21,21,21]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -12729,8 +12729,8 @@ define <4 x i64> @ugt_21_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_21_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [21,21,21,21]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -12812,8 +12812,8 @@ define <4 x i64> @ult_22_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_22_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [22,22,22,22]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -12895,8 +12895,8 @@ define <4 x i64> @ugt_22_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_22_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [22,22,22,22]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -12978,8 +12978,8 @@ define <4 x i64> @ult_23_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_23_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [23,23,23,23]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -13061,8 +13061,8 @@ define <4 x i64> @ugt_23_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_23_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [23,23,23,23]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -13144,8 +13144,8 @@ define <4 x i64> @ult_24_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_24_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [24,24,24,24]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -13227,8 +13227,8 @@ define <4 x i64> @ugt_24_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_24_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [24,24,24,24]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -13310,8 +13310,8 @@ define <4 x i64> @ult_25_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_25_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [25,25,25,25]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -13393,8 +13393,8 @@ define <4 x i64> @ugt_25_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_25_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [25,25,25,25]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -13476,8 +13476,8 @@ define <4 x i64> @ult_26_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_26_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [26,26,26,26]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -13559,8 +13559,8 @@ define <4 x i64> @ugt_26_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_26_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [26,26,26,26]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -13642,8 +13642,8 @@ define <4 x i64> @ult_27_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_27_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [27,27,27,27]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -13725,8 +13725,8 @@ define <4 x i64> @ugt_27_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_27_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [27,27,27,27]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -13808,8 +13808,8 @@ define <4 x i64> @ult_28_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_28_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [28,28,28,28]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -13891,8 +13891,8 @@ define <4 x i64> @ugt_28_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_28_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [28,28,28,28]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -13974,8 +13974,8 @@ define <4 x i64> @ult_29_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_29_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [29,29,29,29]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -14057,8 +14057,8 @@ define <4 x i64> @ugt_29_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_29_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [29,29,29,29]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -14140,8 +14140,8 @@ define <4 x i64> @ult_30_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_30_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [30,30,30,30]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -14223,8 +14223,8 @@ define <4 x i64> @ugt_30_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_30_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [30,30,30,30]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -14306,8 +14306,8 @@ define <4 x i64> @ult_31_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_31_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [31,31,31,31]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -14389,8 +14389,8 @@ define <4 x i64> @ugt_31_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_31_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [31,31,31,31]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -14472,8 +14472,8 @@ define <4 x i64> @ult_32_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_32_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [32,32,32,32]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -14555,8 +14555,8 @@ define <4 x i64> @ugt_32_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_32_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [32,32,32,32]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -14638,8 +14638,8 @@ define <4 x i64> @ult_33_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_33_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [33,33,33,33]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -14721,8 +14721,8 @@ define <4 x i64> @ugt_33_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_33_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [33,33,33,33]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -14804,8 +14804,8 @@ define <4 x i64> @ult_34_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_34_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [34,34,34,34]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -14887,8 +14887,8 @@ define <4 x i64> @ugt_34_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_34_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [34,34,34,34]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -14970,8 +14970,8 @@ define <4 x i64> @ult_35_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_35_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [35,35,35,35]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -15053,8 +15053,8 @@ define <4 x i64> @ugt_35_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_35_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [35,35,35,35]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -15136,8 +15136,8 @@ define <4 x i64> @ult_36_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_36_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [36,36,36,36]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -15219,8 +15219,8 @@ define <4 x i64> @ugt_36_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_36_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [36,36,36,36]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -15302,8 +15302,8 @@ define <4 x i64> @ult_37_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_37_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [37,37,37,37]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -15385,8 +15385,8 @@ define <4 x i64> @ugt_37_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_37_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [37,37,37,37]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -15468,8 +15468,8 @@ define <4 x i64> @ult_38_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_38_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [38,38,38,38]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -15551,8 +15551,8 @@ define <4 x i64> @ugt_38_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_38_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [38,38,38,38]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -15634,8 +15634,8 @@ define <4 x i64> @ult_39_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_39_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [39,39,39,39]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -15717,8 +15717,8 @@ define <4 x i64> @ugt_39_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_39_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [39,39,39,39]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -15800,8 +15800,8 @@ define <4 x i64> @ult_40_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_40_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [40,40,40,40]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -15883,8 +15883,8 @@ define <4 x i64> @ugt_40_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_40_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [40,40,40,40]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -15966,8 +15966,8 @@ define <4 x i64> @ult_41_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_41_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [41,41,41,41]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -16049,8 +16049,8 @@ define <4 x i64> @ugt_41_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_41_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [41,41,41,41]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -16132,8 +16132,8 @@ define <4 x i64> @ult_42_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_42_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [42,42,42,42]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -16215,8 +16215,8 @@ define <4 x i64> @ugt_42_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_42_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [42,42,42,42]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -16298,8 +16298,8 @@ define <4 x i64> @ult_43_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_43_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [43,43,43,43]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -16381,8 +16381,8 @@ define <4 x i64> @ugt_43_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_43_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [43,43,43,43]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -16464,8 +16464,8 @@ define <4 x i64> @ult_44_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_44_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [44,44,44,44]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -16547,8 +16547,8 @@ define <4 x i64> @ugt_44_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_44_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [44,44,44,44]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -16630,8 +16630,8 @@ define <4 x i64> @ult_45_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_45_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [45,45,45,45]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -16713,8 +16713,8 @@ define <4 x i64> @ugt_45_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_45_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [45,45,45,45]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -16796,8 +16796,8 @@ define <4 x i64> @ult_46_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_46_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [46,46,46,46]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -16879,8 +16879,8 @@ define <4 x i64> @ugt_46_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_46_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [46,46,46,46]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -16962,8 +16962,8 @@ define <4 x i64> @ult_47_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_47_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [47,47,47,47]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -17045,8 +17045,8 @@ define <4 x i64> @ugt_47_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_47_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [47,47,47,47]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -17128,8 +17128,8 @@ define <4 x i64> @ult_48_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_48_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [48,48,48,48]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -17211,8 +17211,8 @@ define <4 x i64> @ugt_48_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_48_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [48,48,48,48]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -17294,8 +17294,8 @@ define <4 x i64> @ult_49_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_49_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [49,49,49,49]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -17377,8 +17377,8 @@ define <4 x i64> @ugt_49_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_49_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [49,49,49,49]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -17460,8 +17460,8 @@ define <4 x i64> @ult_50_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_50_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [50,50,50,50]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -17543,8 +17543,8 @@ define <4 x i64> @ugt_50_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_50_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [50,50,50,50]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -17626,8 +17626,8 @@ define <4 x i64> @ult_51_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_51_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [51,51,51,51]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -17709,8 +17709,8 @@ define <4 x i64> @ugt_51_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_51_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [51,51,51,51]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -17792,8 +17792,8 @@ define <4 x i64> @ult_52_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_52_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [52,52,52,52]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -17875,8 +17875,8 @@ define <4 x i64> @ugt_52_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_52_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [52,52,52,52]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -17958,8 +17958,8 @@ define <4 x i64> @ult_53_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_53_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [53,53,53,53]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -18041,8 +18041,8 @@ define <4 x i64> @ugt_53_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_53_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [53,53,53,53]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -18124,8 +18124,8 @@ define <4 x i64> @ult_54_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_54_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [54,54,54,54]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -18207,8 +18207,8 @@ define <4 x i64> @ugt_54_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_54_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [54,54,54,54]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -18290,8 +18290,8 @@ define <4 x i64> @ult_55_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_55_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [55,55,55,55]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -18373,8 +18373,8 @@ define <4 x i64> @ugt_55_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_55_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [55,55,55,55]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -18456,8 +18456,8 @@ define <4 x i64> @ult_56_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_56_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [56,56,56,56]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -18539,8 +18539,8 @@ define <4 x i64> @ugt_56_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_56_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [56,56,56,56]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -18622,8 +18622,8 @@ define <4 x i64> @ult_57_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_57_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [57,57,57,57]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -18705,8 +18705,8 @@ define <4 x i64> @ugt_57_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_57_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [57,57,57,57]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -18788,8 +18788,8 @@ define <4 x i64> @ult_58_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_58_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [58,58,58,58]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -18871,8 +18871,8 @@ define <4 x i64> @ugt_58_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_58_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [58,58,58,58]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -18954,8 +18954,8 @@ define <4 x i64> @ult_59_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_59_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [59,59,59,59]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -19037,8 +19037,8 @@ define <4 x i64> @ugt_59_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_59_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [59,59,59,59]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -19120,8 +19120,8 @@ define <4 x i64> @ult_60_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_60_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [60,60,60,60]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -19203,8 +19203,8 @@ define <4 x i64> @ugt_60_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_60_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [60,60,60,60]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -19286,8 +19286,8 @@ define <4 x i64> @ult_61_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_61_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [61,61,61,61]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -19369,8 +19369,8 @@ define <4 x i64> @ugt_61_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_61_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [61,61,61,61]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -19452,8 +19452,8 @@ define <4 x i64> @ult_62_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_62_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [62,62,62,62]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
@@ -19535,8 +19535,8 @@ define <4 x i64> @ugt_62_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ugt_62_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [62,62,62,62]
; BITALG-NEXT: vpcmpgtq %ymm1, %ymm0, %ymm0
@@ -19618,8 +19618,8 @@ define <4 x i64> @ult_63_v4i64(<4 x i64> %0) {
;
; BITALG-LABEL: ult_63_v4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: vpbroadcastq {{.*#+}} ymm1 = [63,63,63,63]
; BITALG-NEXT: vpcmpgtq %ymm0, %ymm1, %ymm0
diff --git a/llvm/test/CodeGen/X86/vector-popcnt-256.ll b/llvm/test/CodeGen/X86/vector-popcnt-256.ll
index 7fb60b987d95d..a941e7fec5663 100644
--- a/llvm/test/CodeGen/X86/vector-popcnt-256.ll
+++ b/llvm/test/CodeGen/X86/vector-popcnt-256.ll
@@ -90,8 +90,8 @@ define <4 x i64> @testv4i64(<4 x i64> %in) nounwind {
;
; BITALG-LABEL: testv4i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; BITALG-NEXT: retq
%out = call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %in)
@@ -205,8 +205,8 @@ define <8 x i32> @testv8i32(<8 x i32> %in) nounwind {
;
; BITALG-LABEL: testv8i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %ymm0, %ymm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
; BITALG-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
; BITALG-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
diff --git a/llvm/test/CodeGen/X86/vector-popcnt-512-ult-ugt.ll b/llvm/test/CodeGen/X86/vector-popcnt-512-ult-ugt.ll
index 1618a647a4062..5545745c4dc8d 100644
--- a/llvm/test/CodeGen/X86/vector-popcnt-512-ult-ugt.ll
+++ b/llvm/test/CodeGen/X86/vector-popcnt-512-ult-ugt.ll
@@ -3731,8 +3731,8 @@ define <16 x i32> @ugt_2_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_2_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -3812,8 +3812,8 @@ define <16 x i32> @ult_3_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_3_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -3893,8 +3893,8 @@ define <16 x i32> @ugt_3_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_3_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -3974,8 +3974,8 @@ define <16 x i32> @ult_4_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_4_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -4055,8 +4055,8 @@ define <16 x i32> @ugt_4_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_4_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -4136,8 +4136,8 @@ define <16 x i32> @ult_5_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_5_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -4217,8 +4217,8 @@ define <16 x i32> @ugt_5_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_5_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -4298,8 +4298,8 @@ define <16 x i32> @ult_6_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_6_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -4379,8 +4379,8 @@ define <16 x i32> @ugt_6_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_6_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -4460,8 +4460,8 @@ define <16 x i32> @ult_7_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_7_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -4541,8 +4541,8 @@ define <16 x i32> @ugt_7_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_7_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -4622,8 +4622,8 @@ define <16 x i32> @ult_8_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_8_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -4703,8 +4703,8 @@ define <16 x i32> @ugt_8_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_8_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -4784,8 +4784,8 @@ define <16 x i32> @ult_9_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_9_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -4865,8 +4865,8 @@ define <16 x i32> @ugt_9_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_9_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -4946,8 +4946,8 @@ define <16 x i32> @ult_10_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_10_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5027,8 +5027,8 @@ define <16 x i32> @ugt_10_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_10_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5108,8 +5108,8 @@ define <16 x i32> @ult_11_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_11_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5189,8 +5189,8 @@ define <16 x i32> @ugt_11_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_11_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5270,8 +5270,8 @@ define <16 x i32> @ult_12_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_12_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5351,8 +5351,8 @@ define <16 x i32> @ugt_12_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_12_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5432,8 +5432,8 @@ define <16 x i32> @ult_13_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_13_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5513,8 +5513,8 @@ define <16 x i32> @ugt_13_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_13_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5594,8 +5594,8 @@ define <16 x i32> @ult_14_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_14_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5675,8 +5675,8 @@ define <16 x i32> @ugt_14_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_14_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5756,8 +5756,8 @@ define <16 x i32> @ult_15_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_15_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5837,8 +5837,8 @@ define <16 x i32> @ugt_15_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_15_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5918,8 +5918,8 @@ define <16 x i32> @ult_16_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_16_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -5999,8 +5999,8 @@ define <16 x i32> @ugt_16_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_16_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -6080,8 +6080,8 @@ define <16 x i32> @ult_17_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_17_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -6161,8 +6161,8 @@ define <16 x i32> @ugt_17_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_17_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -6242,8 +6242,8 @@ define <16 x i32> @ult_18_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_18_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -6323,8 +6323,8 @@ define <16 x i32> @ugt_18_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_18_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -6404,8 +6404,8 @@ define <16 x i32> @ult_19_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_19_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -6485,8 +6485,8 @@ define <16 x i32> @ugt_19_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_19_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -6566,8 +6566,8 @@ define <16 x i32> @ult_20_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_20_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -6647,8 +6647,8 @@ define <16 x i32> @ugt_20_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_20_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -6728,8 +6728,8 @@ define <16 x i32> @ult_21_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_21_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -6809,8 +6809,8 @@ define <16 x i32> @ugt_21_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_21_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -6890,8 +6890,8 @@ define <16 x i32> @ult_22_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_22_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -6971,8 +6971,8 @@ define <16 x i32> @ugt_22_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_22_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -7052,8 +7052,8 @@ define <16 x i32> @ult_23_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_23_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -7133,8 +7133,8 @@ define <16 x i32> @ugt_23_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_23_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -7214,8 +7214,8 @@ define <16 x i32> @ult_24_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_24_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -7295,8 +7295,8 @@ define <16 x i32> @ugt_24_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_24_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -7376,8 +7376,8 @@ define <16 x i32> @ult_25_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_25_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -7457,8 +7457,8 @@ define <16 x i32> @ugt_25_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_25_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -7538,8 +7538,8 @@ define <16 x i32> @ult_26_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_26_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -7619,8 +7619,8 @@ define <16 x i32> @ugt_26_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_26_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -7700,8 +7700,8 @@ define <16 x i32> @ult_27_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_27_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -7781,8 +7781,8 @@ define <16 x i32> @ugt_27_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_27_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -7862,8 +7862,8 @@ define <16 x i32> @ult_28_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_28_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -7943,8 +7943,8 @@ define <16 x i32> @ugt_28_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_28_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -8024,8 +8024,8 @@ define <16 x i32> @ult_29_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_29_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -8105,8 +8105,8 @@ define <16 x i32> @ugt_29_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_29_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -8186,8 +8186,8 @@ define <16 x i32> @ult_30_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_30_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -8267,8 +8267,8 @@ define <16 x i32> @ugt_30_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ugt_30_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -8348,8 +8348,8 @@ define <16 x i32> @ult_31_v16i32(<16 x i32> %0) {
;
; BITALG-LABEL: ult_31_v16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
@@ -8491,8 +8491,8 @@ define <8 x i64> @ugt_2_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_2_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -8556,8 +8556,8 @@ define <8 x i64> @ult_3_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_3_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -8621,8 +8621,8 @@ define <8 x i64> @ugt_3_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_3_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -8686,8 +8686,8 @@ define <8 x i64> @ult_4_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_4_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -8751,8 +8751,8 @@ define <8 x i64> @ugt_4_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_4_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -8816,8 +8816,8 @@ define <8 x i64> @ult_5_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_5_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -8881,8 +8881,8 @@ define <8 x i64> @ugt_5_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_5_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -8946,8 +8946,8 @@ define <8 x i64> @ult_6_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_6_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9011,8 +9011,8 @@ define <8 x i64> @ugt_6_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_6_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9076,8 +9076,8 @@ define <8 x i64> @ult_7_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_7_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9141,8 +9141,8 @@ define <8 x i64> @ugt_7_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_7_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9206,8 +9206,8 @@ define <8 x i64> @ult_8_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_8_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9271,8 +9271,8 @@ define <8 x i64> @ugt_8_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_8_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9336,8 +9336,8 @@ define <8 x i64> @ult_9_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_9_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9401,8 +9401,8 @@ define <8 x i64> @ugt_9_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_9_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9466,8 +9466,8 @@ define <8 x i64> @ult_10_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_10_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9531,8 +9531,8 @@ define <8 x i64> @ugt_10_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_10_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9596,8 +9596,8 @@ define <8 x i64> @ult_11_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_11_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9661,8 +9661,8 @@ define <8 x i64> @ugt_11_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_11_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9726,8 +9726,8 @@ define <8 x i64> @ult_12_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_12_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9791,8 +9791,8 @@ define <8 x i64> @ugt_12_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_12_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9856,8 +9856,8 @@ define <8 x i64> @ult_13_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_13_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9921,8 +9921,8 @@ define <8 x i64> @ugt_13_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_13_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -9986,8 +9986,8 @@ define <8 x i64> @ult_14_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_14_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10051,8 +10051,8 @@ define <8 x i64> @ugt_14_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_14_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10116,8 +10116,8 @@ define <8 x i64> @ult_15_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_15_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10181,8 +10181,8 @@ define <8 x i64> @ugt_15_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_15_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10246,8 +10246,8 @@ define <8 x i64> @ult_16_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_16_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10311,8 +10311,8 @@ define <8 x i64> @ugt_16_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_16_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10376,8 +10376,8 @@ define <8 x i64> @ult_17_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_17_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10441,8 +10441,8 @@ define <8 x i64> @ugt_17_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_17_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10506,8 +10506,8 @@ define <8 x i64> @ult_18_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_18_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10571,8 +10571,8 @@ define <8 x i64> @ugt_18_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_18_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10636,8 +10636,8 @@ define <8 x i64> @ult_19_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_19_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10701,8 +10701,8 @@ define <8 x i64> @ugt_19_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_19_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10766,8 +10766,8 @@ define <8 x i64> @ult_20_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_20_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10831,8 +10831,8 @@ define <8 x i64> @ugt_20_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_20_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10896,8 +10896,8 @@ define <8 x i64> @ult_21_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_21_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -10961,8 +10961,8 @@ define <8 x i64> @ugt_21_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_21_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11026,8 +11026,8 @@ define <8 x i64> @ult_22_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_22_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11091,8 +11091,8 @@ define <8 x i64> @ugt_22_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_22_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11156,8 +11156,8 @@ define <8 x i64> @ult_23_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_23_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11221,8 +11221,8 @@ define <8 x i64> @ugt_23_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_23_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11286,8 +11286,8 @@ define <8 x i64> @ult_24_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_24_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11351,8 +11351,8 @@ define <8 x i64> @ugt_24_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_24_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11416,8 +11416,8 @@ define <8 x i64> @ult_25_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_25_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11481,8 +11481,8 @@ define <8 x i64> @ugt_25_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_25_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11546,8 +11546,8 @@ define <8 x i64> @ult_26_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_26_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11611,8 +11611,8 @@ define <8 x i64> @ugt_26_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_26_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11676,8 +11676,8 @@ define <8 x i64> @ult_27_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_27_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11741,8 +11741,8 @@ define <8 x i64> @ugt_27_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_27_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11806,8 +11806,8 @@ define <8 x i64> @ult_28_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_28_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11871,8 +11871,8 @@ define <8 x i64> @ugt_28_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_28_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -11936,8 +11936,8 @@ define <8 x i64> @ult_29_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_29_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12001,8 +12001,8 @@ define <8 x i64> @ugt_29_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_29_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12066,8 +12066,8 @@ define <8 x i64> @ult_30_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_30_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12131,8 +12131,8 @@ define <8 x i64> @ugt_30_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_30_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12196,8 +12196,8 @@ define <8 x i64> @ult_31_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_31_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12261,8 +12261,8 @@ define <8 x i64> @ugt_31_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_31_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12326,8 +12326,8 @@ define <8 x i64> @ult_32_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_32_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12391,8 +12391,8 @@ define <8 x i64> @ugt_32_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_32_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12456,8 +12456,8 @@ define <8 x i64> @ult_33_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_33_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12521,8 +12521,8 @@ define <8 x i64> @ugt_33_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_33_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12586,8 +12586,8 @@ define <8 x i64> @ult_34_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_34_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12651,8 +12651,8 @@ define <8 x i64> @ugt_34_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_34_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12716,8 +12716,8 @@ define <8 x i64> @ult_35_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_35_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12781,8 +12781,8 @@ define <8 x i64> @ugt_35_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_35_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12846,8 +12846,8 @@ define <8 x i64> @ult_36_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_36_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12911,8 +12911,8 @@ define <8 x i64> @ugt_36_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_36_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -12976,8 +12976,8 @@ define <8 x i64> @ult_37_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_37_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13041,8 +13041,8 @@ define <8 x i64> @ugt_37_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_37_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13106,8 +13106,8 @@ define <8 x i64> @ult_38_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_38_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13171,8 +13171,8 @@ define <8 x i64> @ugt_38_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_38_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13236,8 +13236,8 @@ define <8 x i64> @ult_39_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_39_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13301,8 +13301,8 @@ define <8 x i64> @ugt_39_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_39_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13366,8 +13366,8 @@ define <8 x i64> @ult_40_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_40_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13431,8 +13431,8 @@ define <8 x i64> @ugt_40_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_40_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13496,8 +13496,8 @@ define <8 x i64> @ult_41_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_41_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13561,8 +13561,8 @@ define <8 x i64> @ugt_41_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_41_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13626,8 +13626,8 @@ define <8 x i64> @ult_42_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_42_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13691,8 +13691,8 @@ define <8 x i64> @ugt_42_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_42_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13756,8 +13756,8 @@ define <8 x i64> @ult_43_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_43_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13821,8 +13821,8 @@ define <8 x i64> @ugt_43_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_43_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13886,8 +13886,8 @@ define <8 x i64> @ult_44_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_44_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -13951,8 +13951,8 @@ define <8 x i64> @ugt_44_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_44_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14016,8 +14016,8 @@ define <8 x i64> @ult_45_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_45_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14081,8 +14081,8 @@ define <8 x i64> @ugt_45_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_45_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14146,8 +14146,8 @@ define <8 x i64> @ult_46_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_46_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14211,8 +14211,8 @@ define <8 x i64> @ugt_46_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_46_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14276,8 +14276,8 @@ define <8 x i64> @ult_47_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_47_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14341,8 +14341,8 @@ define <8 x i64> @ugt_47_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_47_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14406,8 +14406,8 @@ define <8 x i64> @ult_48_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_48_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14471,8 +14471,8 @@ define <8 x i64> @ugt_48_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_48_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14536,8 +14536,8 @@ define <8 x i64> @ult_49_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_49_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14601,8 +14601,8 @@ define <8 x i64> @ugt_49_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_49_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14666,8 +14666,8 @@ define <8 x i64> @ult_50_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_50_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14731,8 +14731,8 @@ define <8 x i64> @ugt_50_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_50_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14796,8 +14796,8 @@ define <8 x i64> @ult_51_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_51_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14861,8 +14861,8 @@ define <8 x i64> @ugt_51_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_51_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14926,8 +14926,8 @@ define <8 x i64> @ult_52_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_52_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -14991,8 +14991,8 @@ define <8 x i64> @ugt_52_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_52_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15056,8 +15056,8 @@ define <8 x i64> @ult_53_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_53_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15121,8 +15121,8 @@ define <8 x i64> @ugt_53_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_53_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15186,8 +15186,8 @@ define <8 x i64> @ult_54_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_54_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15251,8 +15251,8 @@ define <8 x i64> @ugt_54_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_54_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15316,8 +15316,8 @@ define <8 x i64> @ult_55_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_55_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15381,8 +15381,8 @@ define <8 x i64> @ugt_55_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_55_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15446,8 +15446,8 @@ define <8 x i64> @ult_56_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_56_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15511,8 +15511,8 @@ define <8 x i64> @ugt_56_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_56_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15576,8 +15576,8 @@ define <8 x i64> @ult_57_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_57_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15641,8 +15641,8 @@ define <8 x i64> @ugt_57_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_57_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15706,8 +15706,8 @@ define <8 x i64> @ult_58_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_58_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15771,8 +15771,8 @@ define <8 x i64> @ugt_58_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_58_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15836,8 +15836,8 @@ define <8 x i64> @ult_59_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_59_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15901,8 +15901,8 @@ define <8 x i64> @ugt_59_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_59_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -15966,8 +15966,8 @@ define <8 x i64> @ult_60_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_60_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -16031,8 +16031,8 @@ define <8 x i64> @ugt_60_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_60_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -16096,8 +16096,8 @@ define <8 x i64> @ult_61_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_61_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -16161,8 +16161,8 @@ define <8 x i64> @ugt_61_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_61_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -16226,8 +16226,8 @@ define <8 x i64> @ult_62_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_62_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -16291,8 +16291,8 @@ define <8 x i64> @ugt_62_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ugt_62_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpgtq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
@@ -16356,8 +16356,8 @@ define <8 x i64> @ult_63_v8i64(<8 x i64> %0) {
;
; BITALG-LABEL: ult_63_v8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: vpcmpltq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %zmm0, %k1
; BITALG-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
diff --git a/llvm/test/CodeGen/X86/vector-popcnt-512.ll b/llvm/test/CodeGen/X86/vector-popcnt-512.ll
index f470a2be8aee8..79cf6aabbc71e 100644
--- a/llvm/test/CodeGen/X86/vector-popcnt-512.ll
+++ b/llvm/test/CodeGen/X86/vector-popcnt-512.ll
@@ -52,8 +52,8 @@ define <8 x i64> @testv8i64(<8 x i64> %in) nounwind {
;
; BITALG-LABEL: testv8i64:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; BITALG-NEXT: retq
%out = call <8 x i64> @llvm.ctpop.v8i64(<8 x i64> %in)
@@ -119,8 +119,8 @@ define <16 x i32> @testv16i32(<16 x i32> %in) nounwind {
;
; BITALG-LABEL: testv16i32:
; BITALG: # %bb.0:
-; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; BITALG-NEXT: vpopcntb %zmm0, %zmm0
; BITALG-NEXT: vpunpckhdq {{.*#+}} zmm2 = zmm0[2],zmm1[2],zmm0[3],zmm1[3],zmm0[6],zmm1[6],zmm0[7],zmm1[7],zmm0[10],zmm1[10],zmm0[11],zmm1[11],zmm0[14],zmm1[14],zmm0[15],zmm1[15]
; BITALG-NEXT: vpsadbw %zmm1, %zmm2, %zmm2
; BITALG-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll b/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
index 7b846e29ab08d..6a6e2a2c9e13b 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
@@ -2007,8 +2007,8 @@ define i16 @test_v32i16_v32i8(<32 x i16> %a0) nounwind {
; AVX512BW-LABEL: test_v32i16_v32i8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
; AVX512BW-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512BW-NEXT: vpaddq %xmm1, %xmm0, %xmm0
@@ -2022,8 +2022,8 @@ define i16 @test_v32i16_v32i8(<32 x i16> %a0) nounwind {
; AVX512BWVL-LABEL: test_v32i16_v32i8:
; AVX512BWVL: # %bb.0:
; AVX512BWVL-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512BWVL-NEXT: vpmovwb %zmm0, %ymm0
; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpmovwb %zmm0, %ymm0
; AVX512BWVL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; AVX512BWVL-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512BWVL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll b/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll
index 33901a0a71f27..d22b3f5494f2f 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll
@@ -2411,11 +2411,10 @@ define i8 @test_v32i8_v32i1(<32 x i8> %a0) nounwind {
; AVX2: # %bb.0:
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpcmpgtb %ymm0, %ymm1, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $al killed $al killed $eax
@@ -2426,11 +2425,10 @@ define i8 @test_v32i8_v32i1(<32 x i8> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpcmpgtb %ymm0, %ymm1, %ymm0
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX512-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddb %xmm2, %xmm0, %xmm0
; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: # kill: def $al killed $al killed $eax
@@ -2521,8 +2519,7 @@ define i8 @test_v64i8_v64i1(<64 x i8> %a0) nounwind {
; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $al killed $al killed $eax
; AVX2-NEXT: vzeroupper
@@ -2539,8 +2536,7 @@ define i8 @test_v64i8_v64i1(<64 x i8> %a0) nounwind {
; AVX512F-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512F-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: # kill: def $al killed $al killed $eax
; AVX512F-NEXT: vzeroupper
@@ -2557,8 +2553,7 @@ define i8 @test_v64i8_v64i1(<64 x i8> %a0) nounwind {
; AVX512VL-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VL-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: # kill: def $al killed $al killed $eax
; AVX512VL-NEXT: vzeroupper
@@ -2759,8 +2754,7 @@ define i8 @test_v128i8_v128i1(<128 x i8> %a0) nounwind {
; X86-AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: # kill: def $al killed $al killed $eax
; X86-AVX2-NEXT: movl %ebp, %esp
@@ -2782,8 +2776,7 @@ define i8 @test_v128i8_v128i1(<128 x i8> %a0) nounwind {
; X64-AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovd %xmm0, %eax
; X64-AVX2-NEXT: # kill: def $al killed $al killed $eax
; X64-AVX2-NEXT: vzeroupper
@@ -2805,8 +2798,7 @@ define i8 @test_v128i8_v128i1(<128 x i8> %a0) nounwind {
; AVX512F-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512F-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
; AVX512F-NEXT: vmovd %xmm0, %eax
; AVX512F-NEXT: # kill: def $al killed $al killed $eax
; AVX512F-NEXT: vzeroupper
@@ -2828,8 +2820,7 @@ define i8 @test_v128i8_v128i1(<128 x i8> %a0) nounwind {
; AVX512VL-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VL-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: # kill: def $al killed $al killed $eax
; AVX512VL-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll b/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
index 92108c6bd3808..bb9ba6226e762 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
@@ -863,11 +863,11 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
;
; AVX512F-LABEL: test_v64i16_v64i8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpsadbw %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpsadbw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
+; AVX512F-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpaddq %ymm2, %ymm0, %ymm0
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512F-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -879,11 +879,11 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
;
; AVX512VL-LABEL: test_v64i16_v64i8:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512VL-NEXT: vpsadbw %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT: vpsadbw %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; AVX512VL-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
+; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vpaddq %ymm2, %ymm0, %ymm0
; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll b/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll
index be606a3fa62da..2382a8e0ddd14 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll
@@ -940,7 +940,6 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vpmovqw %ymm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
@@ -1212,7 +1211,6 @@ define i32 @reduce_ctpop_v8i32(<8 x i32> %a0) nounwind {
; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
; AVX512VL-NEXT: vpmovdb %ymm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
@@ -1636,7 +1634,6 @@ define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) nounwind {
; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; AVX512VL-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vmovd %xmm0, %eax
; AVX512VL-NEXT: vzeroupper
@@ -2057,44 +2054,83 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: ret{{[l|q]}}
;
-; AVX2-LABEL: reduce_ctpop_v16i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm3
-; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX2-NEXT: # ymm4 = mem[0,1,0,1]
-; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
-; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
-; AVX2-NEXT: vpaddb %ymm3, %ymm1, %ymm1
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT: vpunpckhdq {{.*#+}} ymm5 = ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[6],ymm3[6],ymm1[7],ymm3[7]
-; AVX2-NEXT: vpsadbw %ymm3, %ymm5, %ymm5
-; AVX2-NEXT: vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5]
-; AVX2-NEXT: vpsadbw %ymm3, %ymm1, %ymm1
-; AVX2-NEXT: vpackuswb %ymm5, %ymm1, %ymm1
-; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm5
-; AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm5
-; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
-; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
-; AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm0
-; AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[6],ymm3[6],ymm0[7],ymm3[7]
-; AVX2-NEXT: vpsadbw %ymm3, %ymm2, %ymm2
-; AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]
-; AVX2-NEXT: vpsadbw %ymm3, %ymm0, %ymm0
-; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: ret{{[l|q]}}
+; X86-AVX2-LABEL: reduce_ctpop_v16i32:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX2-NEXT: vpand %ymm2, %ymm1, %ymm4
+; X86-AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX2-NEXT: # ymm3 = mem[0,1,0,1]
+; X86-AVX2-NEXT: vpshufb %ymm4, %ymm3, %ymm4
+; X86-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpshufb %ymm1, %ymm3, %ymm1
+; X86-AVX2-NEXT: vpaddb %ymm4, %ymm1, %ymm4
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpunpckhdq {{.*#+}} ymm5 = ymm4[2],ymm1[2],ymm4[3],ymm1[3],ymm4[6],ymm1[6],ymm4[7],ymm1[7]
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm5, %ymm5
+; X86-AVX2-NEXT: vpunpckldq {{.*#+}} ymm4 = ymm4[0],ymm1[0],ymm4[1],ymm1[1],ymm4[4],ymm1[4],ymm4[5],ymm1[5]
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm4, %ymm4
+; X86-AVX2-NEXT: vpackuswb %ymm5, %ymm4, %ymm4
+; X86-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm5
+; X86-AVX2-NEXT: vpshufb %ymm5, %ymm3, %ymm5
+; X86-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm3, %ymm0
+; X86-AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpaddd %ymm4, %ymm0, %ymm0
+; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X86-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: vzeroupper
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: reduce_ctpop_v16i32:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX2-NEXT: vpand %ymm2, %ymm1, %ymm3
+; X64-AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX2-NEXT: # ymm4 = mem[0,1,0,1]
+; X64-AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3
+; X64-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
+; X64-AVX2-NEXT: vpaddb %ymm3, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X64-AVX2-NEXT: vpunpckhdq {{.*#+}} ymm5 = ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[6],ymm3[6],ymm1[7],ymm3[7]
+; X64-AVX2-NEXT: vpsadbw %ymm3, %ymm5, %ymm5
+; X64-AVX2-NEXT: vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5]
+; X64-AVX2-NEXT: vpsadbw %ymm3, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpackuswb %ymm5, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm5
+; X64-AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm5
+; X64-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
+; X64-AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[6],ymm3[6],ymm0[7],ymm3[7]
+; X64-AVX2-NEXT: vpsadbw %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]
+; X64-AVX2-NEXT: vpsadbw %ymm3, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovd %xmm0, %eax
+; X64-AVX2-NEXT: vzeroupper
+; X64-AVX2-NEXT: retq
;
; AVX512VL-LABEL: reduce_ctpop_v16i32:
; AVX512VL: # %bb.0:
@@ -2114,7 +2150,6 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; AVX512VL-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; AVX512VL-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
; AVX512VL-NEXT: vpmovdb %zmm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
@@ -2864,8 +2899,7 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; AVX512VL-NEXT: vpmovqb %zmm1, %xmm1
; AVX512VL-NEXT: vpmovqb %zmm0, %xmm0
; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vmovd %xmm0, %eax
@@ -5250,59 +5284,59 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; AVX512VL-NEXT: vpsrlw $4, %ymm0, %ymm0
; AVX512VL-NEXT: vpand %ymm0, %ymm8, %ymm0
; AVX512VL-NEXT: vpshufb %ymm0, %ymm10, %ymm0
-; AVX512VL-NEXT: vpaddb %ymm0, %ymm9, %ymm0
-; AVX512VL-NEXT: vpxor %xmm9, %xmm9, %xmm9
-; AVX512VL-NEXT: vpsadbw %ymm0, %ymm9, %ymm0
+; AVX512VL-NEXT: vpaddb %ymm0, %ymm9, %ymm9
+; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsadbw %ymm0, %ymm9, %ymm9
; AVX512VL-NEXT: vpand %ymm1, %ymm8, %ymm11
; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm11
; AVX512VL-NEXT: vpsrlw $4, %ymm1, %ymm1
; AVX512VL-NEXT: vpand %ymm1, %ymm8, %ymm1
; AVX512VL-NEXT: vpshufb %ymm1, %ymm10, %ymm1
; AVX512VL-NEXT: vpaddb %ymm1, %ymm11, %ymm1
-; AVX512VL-NEXT: vpsadbw %ymm1, %ymm9, %ymm1
+; AVX512VL-NEXT: vpsadbw %ymm0, %ymm1, %ymm1
; AVX512VL-NEXT: vpand %ymm2, %ymm8, %ymm11
; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm11
; AVX512VL-NEXT: vpsrlw $4, %ymm2, %ymm2
; AVX512VL-NEXT: vpand %ymm2, %ymm8, %ymm2
; AVX512VL-NEXT: vpshufb %ymm2, %ymm10, %ymm2
; AVX512VL-NEXT: vpaddb %ymm2, %ymm11, %ymm2
-; AVX512VL-NEXT: vpsadbw %ymm2, %ymm9, %ymm2
+; AVX512VL-NEXT: vpsadbw %ymm0, %ymm2, %ymm2
; AVX512VL-NEXT: vpand %ymm3, %ymm8, %ymm11
; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm11
; AVX512VL-NEXT: vpsrlw $4, %ymm3, %ymm3
; AVX512VL-NEXT: vpand %ymm3, %ymm8, %ymm3
; AVX512VL-NEXT: vpshufb %ymm3, %ymm10, %ymm3
; AVX512VL-NEXT: vpaddb %ymm3, %ymm11, %ymm3
-; AVX512VL-NEXT: vpsadbw %ymm3, %ymm9, %ymm3
+; AVX512VL-NEXT: vpsadbw %ymm0, %ymm3, %ymm3
; AVX512VL-NEXT: vpand %ymm4, %ymm8, %ymm11
; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm11
; AVX512VL-NEXT: vpsrlw $4, %ymm4, %ymm4
; AVX512VL-NEXT: vpand %ymm4, %ymm8, %ymm4
; AVX512VL-NEXT: vpshufb %ymm4, %ymm10, %ymm4
; AVX512VL-NEXT: vpaddb %ymm4, %ymm11, %ymm4
-; AVX512VL-NEXT: vpsadbw %ymm4, %ymm9, %ymm4
+; AVX512VL-NEXT: vpsadbw %ymm0, %ymm4, %ymm4
; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm11
; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm11
; AVX512VL-NEXT: vpsrlw $4, %ymm5, %ymm5
; AVX512VL-NEXT: vpand %ymm5, %ymm8, %ymm5
; AVX512VL-NEXT: vpshufb %ymm5, %ymm10, %ymm5
; AVX512VL-NEXT: vpaddb %ymm5, %ymm11, %ymm5
-; AVX512VL-NEXT: vpsadbw %ymm5, %ymm9, %ymm5
+; AVX512VL-NEXT: vpsadbw %ymm0, %ymm5, %ymm5
; AVX512VL-NEXT: vpand %ymm6, %ymm8, %ymm11
; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm11
; AVX512VL-NEXT: vpsrlw $4, %ymm6, %ymm6
; AVX512VL-NEXT: vpand %ymm6, %ymm8, %ymm6
; AVX512VL-NEXT: vpshufb %ymm6, %ymm10, %ymm6
; AVX512VL-NEXT: vpaddb %ymm6, %ymm11, %ymm6
-; AVX512VL-NEXT: vpsadbw %ymm6, %ymm9, %ymm6
+; AVX512VL-NEXT: vpsadbw %ymm0, %ymm6, %ymm6
; AVX512VL-NEXT: vpand %ymm7, %ymm8, %ymm11
; AVX512VL-NEXT: vpshufb %ymm11, %ymm10, %ymm11
; AVX512VL-NEXT: vpsrlw $4, %ymm7, %ymm7
; AVX512VL-NEXT: vpand %ymm7, %ymm8, %ymm7
; AVX512VL-NEXT: vpshufb %ymm7, %ymm10, %ymm7
; AVX512VL-NEXT: vpaddb %ymm7, %ymm11, %ymm7
-; AVX512VL-NEXT: vpsadbw %ymm7, %ymm9, %ymm7
-; AVX512VL-NEXT: vpmovqw %ymm0, %xmm0
+; AVX512VL-NEXT: vpsadbw %ymm0, %ymm7, %ymm7
+; AVX512VL-NEXT: vpmovqw %ymm9, %xmm8
; AVX512VL-NEXT: vpmovqw %ymm1, %xmm1
; AVX512VL-NEXT: vpmovqw %ymm2, %xmm2
; AVX512VL-NEXT: vpmovqw %ymm3, %xmm3
@@ -5313,14 +5347,13 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; AVX512VL-NEXT: vinserti128 $1, %xmm5, %ymm4, %ymm4
; AVX512VL-NEXT: vinserti128 $1, %xmm7, %ymm6, %ymm5
; AVX512VL-NEXT: vinserti64x4 $1, %ymm4, %zmm5, %zmm4
-; AVX512VL-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX512VL-NEXT: vpsadbw %zmm5, %zmm4, %zmm4
-; AVX512VL-NEXT: vpmovsxbd {{.*#+}} ymm6 = [8,12,8,12,8,12,0,4]
-; AVX512VL-NEXT: vpermd %zmm4, %zmm6, %zmm4
-; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm1
-; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512VL-NEXT: vpsadbw %zmm5, %zmm0, %zmm0
+; AVX512VL-NEXT: vpsadbw %zmm0, %zmm4, %zmm4
+; AVX512VL-NEXT: vpmovsxbd {{.*#+}} ymm5 = [8,12,8,12,8,12,0,4]
+; AVX512VL-NEXT: vpermd %zmm4, %zmm5, %zmm4
+; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm8, %ymm1
+; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX512VL-NEXT: vinserti64x4 $1, %ymm1, %zmm2, %zmm1
+; AVX512VL-NEXT: vpsadbw %zmm0, %zmm1, %zmm0
; AVX512VL-NEXT: vpmovsxbd {{.*#+}} xmm1 = [8,12,0,4]
; AVX512VL-NEXT: vpermd %zmm0, %zmm1, %zmm0
; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
diff --git a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
index e038419fae246..cee2c34cbb8e2 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-or-bool.ll
@@ -1743,11 +1743,11 @@ define i1 @icmp0_v32i16_v32i1(<32 x i16>) nounwind {
;
; AVX512F-LABEL: icmp0_v32i16_v32i1:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT: vpcmpeqw %ymm1, %ymm2, %ymm2
+; AVX512F-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpor %ymm2, %ymm0, %ymm0
; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm0
; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k0
; AVX512F-NEXT: kortestw %k0, %k0
@@ -1844,11 +1844,11 @@ define i1 @icmp0_v64i8_v64i1(<64 x i8>) nounwind {
;
; AVX512F-LABEL: icmp0_v64i8_v64i1:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT: vpor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT: vpcmpeqb %ymm1, %ymm2, %ymm2
+; AVX512F-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpor %ymm2, %ymm0, %ymm0
; AVX512F-NEXT: vpmovmskb %ymm0, %eax
; AVX512F-NEXT: testl %eax, %eax
; AVX512F-NEXT: setne %al
diff --git a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
index c9b1f2a5edaa0..e0871ccb02242 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-xor-bool.ll
@@ -1980,11 +1980,11 @@ define i1 @icmp0_v32i16_v32i1(<32 x i16>) nounwind {
;
; AVX512F-LABEL: icmp0_v32i16_v32i1:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT: vpxor %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT: vpcmpeqw %ymm1, %ymm2, %ymm2
+; AVX512F-NEXT: vpcmpeqw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpxor %ymm2, %ymm0, %ymm0
; AVX512F-NEXT: vpmovsxwd %ymm0, %zmm0
; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k0
; AVX512F-NEXT: kshiftrw $8, %k0, %k1
@@ -2100,15 +2100,15 @@ define i1 @icmp0_v64i8_v64i1(<64 x i8>) nounwind {
;
; AVX512F-LABEL: icmp0_v64i8_v64i1:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vpcmpeqb %ymm2, %ymm0, %ymm0
-; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm2
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; AVX512F-NEXT: vpcmpeqb %ymm1, %ymm2, %ymm2
+; AVX512F-NEXT: vpcmpeqb %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vextracti128 $1, %ymm2, %xmm1
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm3
-; AVX512F-NEXT: vpxor %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT: vpxor %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpxor %xmm1, %xmm3, %xmm1
; AVX512F-NEXT: vpxor %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT: vpxor %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpmovsxbd %xmm0, %zmm0
; AVX512F-NEXT: vpslld $31, %zmm0, %zmm0
; AVX512F-NEXT: vptestmd %zmm0, %zmm0, %k0
diff --git a/llvm/test/CodeGen/X86/vector-rotate-256.ll b/llvm/test/CodeGen/X86/vector-rotate-256.ll
index 48083ac9e810a..1cca64bcdfb6a 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-256.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-256.ll
@@ -208,8 +208,8 @@ define <16 x i16> @var_rotate_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
;
; AVX2-LABEL: var_rotate_v16i16:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
; AVX2-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
; AVX2-NEXT: vpsllvd %ymm3, %ymm4, %ymm3
@@ -223,8 +223,8 @@ define <16 x i16> @var_rotate_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
;
; AVX512F-LABEL: var_rotate_v16i16:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512F-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
; AVX512F-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
; AVX512F-NEXT: vpsllvd %ymm3, %ymm4, %ymm3
@@ -238,8 +238,8 @@ define <16 x i16> @var_rotate_v16i16(<16 x i16> %a, <16 x i16> %b) nounwind {
;
; AVX512VL-LABEL: var_rotate_v16i16:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VL-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
; AVX512VL-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm0[4,4,5,5,6,6,7,7,12,12,13,13,14,14,15,15]
; AVX512VL-NEXT: vpsllvd %ymm3, %ymm4, %ymm3
@@ -427,8 +427,8 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX512BW-LABEL: var_rotate_v32i8:
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512BW-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]
; AVX512BW-NEXT: vpsllvw %zmm4, %zmm2, %zmm2
; AVX512BW-NEXT: vpsrlw $8, %ymm2, %ymm2
@@ -441,8 +441,8 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
;
; AVX512VLBW-LABEL: var_rotate_v32i8:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; AVX512VLBW-NEXT: vpsllvw %ymm3, %ymm4, %ymm3
@@ -457,8 +457,8 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
; AVX512VBMI2-LABEL: var_rotate_v32i8:
; AVX512VBMI2: # %bb.0:
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} ymm2 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
-; AVX512VBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VBMI2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512VBMI2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm1[8],ymm3[8],ymm1[9],ymm3[9],ymm1[10],ymm3[10],ymm1[11],ymm3[11],ymm1[12],ymm3[12],ymm1[13],ymm3[13],ymm1[14],ymm3[14],ymm1[15],ymm3[15],ymm1[24],ymm3[24],ymm1[25],ymm3[25],ymm1[26],ymm3[26],ymm1[27],ymm3[27],ymm1[28],ymm3[28],ymm1[29],ymm3[29],ymm1[30],ymm3[30],ymm1[31],ymm3[31]
; AVX512VBMI2-NEXT: vpsllvw %zmm4, %zmm2, %zmm2
; AVX512VBMI2-NEXT: vpsrlw $8, %ymm2, %ymm2
@@ -471,8 +471,8 @@ define <32 x i8> @var_rotate_v32i8(<32 x i8> %a, <32 x i8> %b) nounwind {
;
; AVX512VLVBMI2-LABEL: var_rotate_v32i8:
; AVX512VLVBMI2: # %bb.0:
-; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VLVBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} ymm4 = ymm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31]
; AVX512VLVBMI2-NEXT: vpsllvw %ymm3, %ymm4, %ymm3
diff --git a/llvm/test/CodeGen/X86/vector-rotate-512.ll b/llvm/test/CodeGen/X86/vector-rotate-512.ll
index cea29414fe808..76ce4923a043b 100644
--- a/llvm/test/CodeGen/X86/vector-rotate-512.ll
+++ b/llvm/test/CodeGen/X86/vector-rotate-512.ll
@@ -220,8 +220,8 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
;
; AVX512BW-LABEL: var_rotate_v64i8:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; AVX512BW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; AVX512BW-NEXT: vpsllvw %zmm3, %zmm4, %zmm3
@@ -235,8 +235,8 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
;
; AVX512VLBW-LABEL: var_rotate_v64i8:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; AVX512VLBW-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; AVX512VLBW-NEXT: vpsllvw %zmm3, %zmm4, %zmm3
@@ -250,8 +250,8 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
;
; AVX512VBMI2-LABEL: var_rotate_v64i8:
; AVX512VBMI2: # %bb.0:
-; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; AVX512VBMI2-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; AVX512VBMI2-NEXT: vpsllvw %zmm3, %zmm4, %zmm3
@@ -265,8 +265,8 @@ define <64 x i8> @var_rotate_v64i8(<64 x i8> %a, <64 x i8> %b) nounwind {
;
; AVX512VLVBMI2-LABEL: var_rotate_v64i8:
; AVX512VLVBMI2: # %bb.0:
-; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VLVBMI2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLVBMI2-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm3 = zmm1[8],zmm2[8],zmm1[9],zmm2[9],zmm1[10],zmm2[10],zmm1[11],zmm2[11],zmm1[12],zmm2[12],zmm1[13],zmm2[13],zmm1[14],zmm2[14],zmm1[15],zmm2[15],zmm1[24],zmm2[24],zmm1[25],zmm2[25],zmm1[26],zmm2[26],zmm1[27],zmm2[27],zmm1[28],zmm2[28],zmm1[29],zmm2[29],zmm1[30],zmm2[30],zmm1[31],zmm2[31],zmm1[40],zmm2[40],zmm1[41],zmm2[41],zmm1[42],zmm2[42],zmm1[43],zmm2[43],zmm1[44],zmm2[44],zmm1[45],zmm2[45],zmm1[46],zmm2[46],zmm1[47],zmm2[47],zmm1[56],zmm2[56],zmm1[57],zmm2[57],zmm1[58],zmm2[58],zmm1[59],zmm2[59],zmm1[60],zmm2[60],zmm1[61],zmm2[61],zmm1[62],zmm2[62],zmm1[63],zmm2[63]
; AVX512VLVBMI2-NEXT: vpunpckhbw {{.*#+}} zmm4 = zmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15,24,24,25,25,26,26,27,27,28,28,29,29,30,30,31,31,40,40,41,41,42,42,43,43,44,44,45,45,46,46,47,47,56,56,57,57,58,58,59,59,60,60,61,61,62,62,63,63]
; AVX512VLVBMI2-NEXT: vpsllvw %zmm3, %zmm4, %zmm3
diff --git a/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll b/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll
index 4450d07e01cca..e3fe589aba617 100644
--- a/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll
+++ b/llvm/test/CodeGen/X86/vector-shift-lshr-512.ll
@@ -351,27 +351,27 @@ define <64 x i8> @constant_shift_v64i8_quads(<64 x i8> %a) nounwind {
define <64 x i8> @constant_shift_v64i8(<64 x i8> %a) nounwind {
; AVX512DQ-LABEL: constant_shift_v64i8:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512DQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15],ymm1[24],ymm2[24],ymm1[25],ymm2[25],ymm1[26],ymm2[26],ymm1[27],ymm2[27],ymm1[28],ymm2[28],ymm1[29],ymm2[29],ymm1[30],ymm2[30],ymm1[31],ymm2[31]
+; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm2
+; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm2[8],ymm1[8],ymm2[9],ymm1[9],ymm2[10],ymm1[10],ymm2[11],ymm1[11],ymm2[12],ymm1[12],ymm2[13],ymm1[13],ymm2[14],ymm1[14],ymm2[15],ymm1[15],ymm2[24],ymm1[24],ymm2[25],ymm1[25],ymm2[26],ymm1[26],ymm2[27],ymm1[27],ymm2[28],ymm1[28],ymm2[29],ymm1[29],ymm2[30],ymm1[30],ymm2[31],ymm1[31]
; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [2,4,8,16,32,64,128,256,2,4,8,16,32,64,128,256]
; AVX512DQ-NEXT: # ymm4 = mem[0,1,0,1]
; AVX512DQ-NEXT: vpmullw %ymm4, %ymm3, %ymm3
; AVX512DQ-NEXT: vpsrlw $8, %ymm3, %ymm3
-; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[16],ymm2[16],ymm1[17],ymm2[17],ymm1[18],ymm2[18],ymm1[19],ymm2[19],ymm1[20],ymm2[20],ymm1[21],ymm2[21],ymm1[22],ymm2[22],ymm1[23],ymm2[23]
+; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm2 = ymm2[0],ymm1[0],ymm2[1],ymm1[1],ymm2[2],ymm1[2],ymm2[3],ymm1[3],ymm2[4],ymm1[4],ymm2[5],ymm1[5],ymm2[6],ymm1[6],ymm2[7],ymm1[7],ymm2[16],ymm1[16],ymm2[17],ymm1[17],ymm2[18],ymm1[18],ymm2[19],ymm1[19],ymm2[20],ymm1[20],ymm2[21],ymm1[21],ymm2[22],ymm1[22],ymm2[23],ymm1[23]
; AVX512DQ-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [256,128,64,32,16,8,4,2,256,128,64,32,16,8,4,2]
; AVX512DQ-NEXT: # ymm5 = mem[0,1,0,1]
-; AVX512DQ-NEXT: vpmullw %ymm5, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpsrlw $8, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpackuswb %ymm3, %ymm1, %ymm1
-; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm2[8],ymm0[9],ymm2[9],ymm0[10],ymm2[10],ymm0[11],ymm2[11],ymm0[12],ymm2[12],ymm0[13],ymm2[13],ymm0[14],ymm2[14],ymm0[15],ymm2[15],ymm0[24],ymm2[24],ymm0[25],ymm2[25],ymm0[26],ymm2[26],ymm0[27],ymm2[27],ymm0[28],ymm2[28],ymm0[29],ymm2[29],ymm0[30],ymm2[30],ymm0[31],ymm2[31]
+; AVX512DQ-NEXT: vpmullw %ymm5, %ymm2, %ymm2
+; AVX512DQ-NEXT: vpsrlw $8, %ymm2, %ymm2
+; AVX512DQ-NEXT: vpackuswb %ymm3, %ymm2, %ymm2
+; AVX512DQ-NEXT: vpunpckhbw {{.*#+}} ymm3 = ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15],ymm0[24],ymm1[24],ymm0[25],ymm1[25],ymm0[26],ymm1[26],ymm0[27],ymm1[27],ymm0[28],ymm1[28],ymm0[29],ymm1[29],ymm0[30],ymm1[30],ymm0[31],ymm1[31]
; AVX512DQ-NEXT: vpmullw %ymm4, %ymm3, %ymm3
; AVX512DQ-NEXT: vpsrlw $8, %ymm3, %ymm3
-; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[1],ymm2[1],ymm0[2],ymm2[2],ymm0[3],ymm2[3],ymm0[4],ymm2[4],ymm0[5],ymm2[5],ymm0[6],ymm2[6],ymm0[7],ymm2[7],ymm0[16],ymm2[16],ymm0[17],ymm2[17],ymm0[18],ymm2[18],ymm0[19],ymm2[19],ymm0[20],ymm2[20],ymm0[21],ymm2[21],ymm0[22],ymm2[22],ymm0[23],ymm2[23]
+; AVX512DQ-NEXT: vpunpcklbw {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[16],ymm1[16],ymm0[17],ymm1[17],ymm0[18],ymm1[18],ymm0[19],ymm1[19],ymm0[20],ymm1[20],ymm0[21],ymm1[21],ymm0[22],ymm1[22],ymm0[23],ymm1[23]
; AVX512DQ-NEXT: vpmullw %ymm5, %ymm0, %ymm0
; AVX512DQ-NEXT: vpsrlw $8, %ymm0, %ymm0
; AVX512DQ-NEXT: vpackuswb %ymm3, %ymm0, %ymm0
-; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
+; AVX512DQ-NEXT: vinserti64x4 $1, %ymm2, %zmm0, %zmm0
; AVX512DQ-NEXT: retq
;
; AVX512BW-LABEL: constant_shift_v64i8:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
index 777d21e5ccd19..dbce67dc9127e 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
@@ -3090,9 +3090,9 @@ define <16 x i16> @shuffle_v16i16_28_zz_zz_zz_29_zz_zz_zz_30_zz_zz_zz_31_zz_zz_z
;
; AVX512VL-LABEL: shuffle_v16i16_28_zz_zz_zz_29_zz_zz_zz_30_zz_zz_zz_31_zz_zz_zz:
; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpmovsxbw {{.*#+}} ymm2 = [28,1,2,3,29,5,6,7,30,9,10,11,31,13,14,15]
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpermt2w %ymm0, %ymm2, %ymm1
+; AVX512VL-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VL-NEXT: vpmovsxbw {{.*#+}} ymm1 = [28,1,2,3,29,5,6,7,30,9,10,11,31,13,14,15]
+; AVX512VL-NEXT: vpermi2w %ymm0, %ymm2, %ymm1
; AVX512VL-NEXT: vmovdqa %ymm1, %ymm0
; AVX512VL-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
index 9e5aa2e0102bc..a879822d85e1d 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v32.ll
@@ -3901,9 +3901,9 @@ define <32 x i8> @shuffle_v32i8_56_zz_zz_zz_57_zz_zz_zz_58_zz_zz_zz__zz_59_zz_zz
;
; AVX512VLVBMI-LABEL: shuffle_v32i8_56_zz_zz_zz_57_zz_zz_zz_58_zz_zz_zz__zz_59_zz_zz_zz_60_zz_zz_zz_61_zz_zz_zz_62_zz_zz_zz_63_zz_zz_zz:
; AVX512VLVBMI: # %bb.0:
-; AVX512VLVBMI-NEXT: vmovdqa {{.*#+}} ymm2 = [56,1,2,3,57,5,6,7,58,9,10,11,59,13,14,15,60,17,18,19,61,21,22,23,62,25,26,27,63,29,30,31]
-; AVX512VLVBMI-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VLVBMI-NEXT: vpermt2b %ymm0, %ymm2, %ymm1
+; AVX512VLVBMI-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLVBMI-NEXT: vmovdqa {{.*#+}} ymm1 = [56,1,2,3,57,5,6,7,58,9,10,11,59,13,14,15,60,17,18,19,61,21,22,23,62,25,26,27,63,29,30,31]
+; AVX512VLVBMI-NEXT: vpermi2b %ymm0, %ymm2, %ymm1
; AVX512VLVBMI-NEXT: vmovdqa %ymm1, %ymm0
; AVX512VLVBMI-NEXT: retq
;
@@ -3949,10 +3949,10 @@ define <32 x i8> @shuffle_v32i8_01_09_00_03_11_02_05_13_04_07_15_06_17_25_16_19_
;
; AVX512VLBW-LABEL: shuffle_v32i8_01_09_00_03_11_02_05_13_04_07_15_06_17_25_16_19_27_18_21_29_20_23_31_22_zz_zz_zz_zz_zz_zz_zz_zz:
; AVX512VLBW: # %bb.0:
-; AVX512VLBW-NEXT: vpshufb {{.*#+}} ymm1 = ymm0[1,9,0,3,11,2,5,13,4,7,15,6,u,u,u,u,17,25,16,19,27,18,21,29,20,23,31,22,u,u,u,u]
-; AVX512VLBW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512VLBW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512VLBW-NEXT: vpshufb {{.*#+}} ymm2 = ymm0[1,9,0,3,11,2,5,13,4,7,15,6,u,u,u,u,17,25,16,19,27,18,21,29,20,23,31,22,u,u,u,u]
; AVX512VLBW-NEXT: vpmovsxbd {{.*#+}} ymm0 = [0,1,2,4,5,6,14,15]
-; AVX512VLBW-NEXT: vpermi2d %ymm2, %ymm1, %ymm0
+; AVX512VLBW-NEXT: vpermi2d %ymm1, %ymm2, %ymm0
; AVX512VLBW-NEXT: retq
;
; AVX512VLVBMI-LABEL: shuffle_v32i8_01_09_00_03_11_02_05_13_04_07_15_06_17_25_16_19_27_18_21_29_20_23_31_22_zz_zz_zz_zz_zz_zz_zz_zz:
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll b/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
index db3be98efa530..5e3e74da1edf9 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-avx512.ll
@@ -245,9 +245,9 @@ define <16 x float> @expand12(<8 x float> %a) {
; CHECK-LABEL: expand12:
; CHECK: # %bb.0:
; CHECK-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; CHECK-NEXT: vpmovsxbd {{.*#+}} zmm2 = [0,16,2,16,4,16,6,16,0,16,1,16,2,16,3,16]
-; CHECK-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; CHECK-NEXT: vpermt2ps %zmm0, %zmm2, %zmm1
+; CHECK-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; CHECK-NEXT: vpmovsxbd {{.*#+}} zmm1 = [0,16,2,16,4,16,6,16,0,16,1,16,2,16,3,16]
+; CHECK-NEXT: vpermi2ps %zmm0, %zmm2, %zmm1
; CHECK-NEXT: vmovaps %zmm1, %zmm0
; CHECK-NEXT: ret{{[l|q]}}
%res = shufflevector <8 x float> zeroinitializer, <8 x float> %a, <16 x i32> <i32 0, i32 8, i32 1, i32 8, i32 2, i32 8, i32 3, i32 8,i32 0, i32 8, i32 1, i32 8, i32 2, i32 8, i32 3, i32 8>
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-v1.ll b/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
index 9645f7c524cb4..8e4e42fb3d13c 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-v1.ll
@@ -674,9 +674,9 @@ define i8 @shuf8i1_9_6_1_10_3_7_7_0(i8 %a) {
; AVX512F: # %bb.0:
; AVX512F-NEXT: kmovw %edi, %k1
; AVX512F-NEXT: vpternlogq {{.*#+}} zmm0 {%k1} {z} = -1
-; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm1 = [9,1,2,10,4,5,6,7]
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm2 = [9,1,2,10,4,5,6,7]
+; AVX512F-NEXT: vpermi2q %zmm0, %zmm1, %zmm2
; AVX512F-NEXT: vptestmq %zmm2, %zmm2, %k0
; AVX512F-NEXT: kmovw %k0, %eax
; AVX512F-NEXT: # kill: def $al killed $al killed $eax
diff --git a/llvm/test/CodeGen/X86/vmaskmov-offset.ll b/llvm/test/CodeGen/X86/vmaskmov-offset.ll
index 73813c0106e09..e0791f9fe2779 100644
--- a/llvm/test/CodeGen/X86/vmaskmov-offset.ll
+++ b/llvm/test/CodeGen/X86/vmaskmov-offset.ll
@@ -11,9 +11,10 @@ define void @test_v16f(<16 x i32> %x) {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vr256 = COPY $ymm1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr256 = COPY $ymm0
- ; CHECK-NEXT: [[AVX_SET0_:%[0-9]+]]:vr256 = AVX_SET0
- ; CHECK-NEXT: [[VPCMPEQDYrr:%[0-9]+]]:vr256 = VPCMPEQDYrr [[COPY]], [[AVX_SET0_]]
- ; CHECK-NEXT: [[VPCMPEQDYrr1:%[0-9]+]]:vr256 = VPCMPEQDYrr [[COPY1]], [[AVX_SET0_]]
+ ; CHECK-NEXT: [[V_SET0_:%[0-9]+]]:vr128 = V_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:vr256 = SUBREG_TO_REG killed [[V_SET0_]], %subreg.sub_xmm
+ ; CHECK-NEXT: [[VPCMPEQDYrr:%[0-9]+]]:vr256 = VPCMPEQDYrr [[COPY]], [[SUBREG_TO_REG]]
+ ; CHECK-NEXT: [[VPCMPEQDYrr1:%[0-9]+]]:vr256 = VPCMPEQDYrr [[COPY1]], [[SUBREG_TO_REG]]
; CHECK-NEXT: [[VMASKMOVPSYrm:%[0-9]+]]:vr256 = VMASKMOVPSYrm [[VPCMPEQDYrr1]], %stack.0.stack_input_vec, 1, $noreg, 0, $noreg :: (load unknown-size from %ir.stack_input_vec, align 4)
; CHECK-NEXT: [[VMASKMOVPSYrm1:%[0-9]+]]:vr256 = VMASKMOVPSYrm [[VPCMPEQDYrr]], %stack.0.stack_input_vec, 1, $noreg, 32, $noreg :: (load unknown-size from %ir.stack_input_vec + 32, align 4)
; CHECK-NEXT: VMASKMOVPSYmr %stack.1.stack_output_vec, 1, $noreg, 32, $noreg, [[VPCMPEQDYrr]], killed [[VMASKMOVPSYrm1]] :: (store unknown-size into %ir.stack_output_vec + 32, align 4)
@@ -38,9 +39,10 @@ define void @test_v8d(<8 x i64> %x) {
; CHECK-NEXT: {{ $}}
; CHECK-NEXT: [[COPY:%[0-9]+]]:vr256 = COPY $ymm1
; CHECK-NEXT: [[COPY1:%[0-9]+]]:vr256 = COPY $ymm0
- ; CHECK-NEXT: [[AVX_SET0_:%[0-9]+]]:vr256 = AVX_SET0
- ; CHECK-NEXT: [[VPCMPEQQYrr:%[0-9]+]]:vr256 = VPCMPEQQYrr [[COPY]], [[AVX_SET0_]]
- ; CHECK-NEXT: [[VPCMPEQQYrr1:%[0-9]+]]:vr256 = VPCMPEQQYrr [[COPY1]], [[AVX_SET0_]]
+ ; CHECK-NEXT: [[V_SET0_:%[0-9]+]]:vr128 = V_SET0
+ ; CHECK-NEXT: [[SUBREG_TO_REG:%[0-9]+]]:vr256 = SUBREG_TO_REG killed [[V_SET0_]], %subreg.sub_xmm
+ ; CHECK-NEXT: [[VPCMPEQQYrr:%[0-9]+]]:vr256 = VPCMPEQQYrr [[COPY]], [[SUBREG_TO_REG]]
+ ; CHECK-NEXT: [[VPCMPEQQYrr1:%[0-9]+]]:vr256 = VPCMPEQQYrr [[COPY1]], [[SUBREG_TO_REG]]
; CHECK-NEXT: [[VMASKMOVPDYrm:%[0-9]+]]:vr256 = VMASKMOVPDYrm [[VPCMPEQQYrr1]], %stack.0.stack_input_vec, 1, $noreg, 0, $noreg :: (load unknown-size from %ir.stack_input_vec, align 4)
; CHECK-NEXT: [[VMASKMOVPDYrm1:%[0-9]+]]:vr256 = VMASKMOVPDYrm [[VPCMPEQQYrr]], %stack.0.stack_input_vec, 1, $noreg, 32, $noreg :: (load unknown-size from %ir.stack_input_vec + 32, align 4)
; CHECK-NEXT: VMASKMOVPDYmr %stack.1.stack_output_vec, 1, $noreg, 32, $noreg, [[VPCMPEQQYrr]], killed [[VMASKMOVPDYrm1]] :: (store unknown-size into %ir.stack_output_vec + 32, align 4)
diff --git a/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll b/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
index e9ddc576c6cd8..e6e20d03c4138 100644
--- a/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
+++ b/llvm/test/CodeGen/X86/wide-scalar-shift-by-byte-multiple-legalization.ll
@@ -5249,8 +5249,8 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups (%rdi), %ymm0
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %ecx
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: leal (,%rcx,8), %eax
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: andb $24, %cl
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: negb %cl
@@ -5296,8 +5296,8 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: vmovups (%rdi), %ymm0
; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %eax
; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: leal (,%rax,8), %ecx
-; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: andb $24, %al
; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: negb %al
@@ -5324,8 +5324,8 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%rdi), %ymm0
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%rsi), %esi
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: leal (,%rsi,8), %eax
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $24, %sil
@@ -5360,8 +5360,8 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vmovups (%rdi), %ymm0
; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%rsi), %eax
; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: leal (,%rax,8), %ecx
-; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: andb $24, %al
; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: negb %al
@@ -6119,8 +6119,8 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%eax), %ecx
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: movb %cl, %dh
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $3, %dh
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: andb $28, %cl
; X86-NO-SHLD-NO-BMI2-AVX-NEXT: negb %cl
@@ -6229,8 +6229,8 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT: movzbl (%eax), %eax
; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT: shlb $3, %cl
-; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT: andb $28, %al
; X86-HAVE-SHLD-NO-BMI2-AVX-NEXT: negb %al
@@ -6291,8 +6291,8 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%eax), %edx
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %edx, %eax
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $3, %al
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ebx
; X86-NO-SHLD-HAVE-BMI2-AVX-NEXT: andb $28, %dl
@@ -6377,8 +6377,8 @@ define void @shl_32bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%eax), %eax
; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: shlb $3, %cl
-; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: andb $28, %al
; X86-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: negb %al
@@ -6762,8 +6762,8 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %ecx
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: movl %ecx, %eax
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $5, %al
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: shlb $2, %cl
; X64-NO-SHLD-NO-BMI2-AVX-NEXT: andb $24, %cl
@@ -6811,8 +6811,8 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: movzbl (%rsi), %eax
; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: movl %eax, %ecx
; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: shlb $5, %cl
-; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: shlb $2, %al
; X64-HAVE-SHLD-NO-BMI2-AVX-NEXT: andb $24, %al
@@ -6840,8 +6840,8 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%rsi), %esi
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %esi, %eax
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $5, %al
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
; X64-NO-SHLD-HAVE-BMI2-AVX-NEXT: shlb $2, %sil
@@ -6878,8 +6878,8 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: movzbl (%rsi), %eax
; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: movl %eax, %ecx
; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: shlb $5, %cl
-; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: shlb $2, %al
; X64-HAVE-SHLD-HAVE-BMI2-AVX-NEXT: andb $24, %al
@@ -6999,9 +6999,9 @@ define void @shl_32bytes_dwordOff(ptr %src.ptr, ptr %dwordOff.ptr, ptr %dst) nou
; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-AVX-NEXT: vmovups (%edx), %ymm0
; X86-AVX-NEXT: movzbl (%ecx), %ecx
-; X86-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X86-AVX-NEXT: vmovups %ymm1, (%esp)
; X86-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86-AVX-NEXT: vmovups %ymm0, (%esp)
; X86-AVX-NEXT: shlb $2, %cl
; X86-AVX-NEXT: andb $28, %cl
; X86-AVX-NEXT: negb %cl
@@ -7074,8 +7074,8 @@ define void @shl_32bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) nou
; X64-AVX: # %bb.0:
; X64-AVX-NEXT: vmovups (%rdi), %ymm0
; X64-AVX-NEXT: movzbl (%rsi), %eax
-; X64-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X64-AVX-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
; X64-AVX-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-AVX-NEXT: shlb $3, %al
; X64-AVX-NEXT: andb $24, %al
@@ -7186,9 +7186,9 @@ define void @shl_32bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) nou
; X86-AVX-NEXT: movl {{[0-9]+}}(%esp), %edx
; X86-AVX-NEXT: vmovups (%edx), %ymm0
; X86-AVX-NEXT: movzbl (%ecx), %ecx
-; X86-AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; X86-AVX-NEXT: vmovups %ymm1, (%esp)
; X86-AVX-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-AVX-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86-AVX-NEXT: vmovups %ymm0, (%esp)
; X86-AVX-NEXT: shlb $3, %cl
; X86-AVX-NEXT: andb $24, %cl
; X86-AVX-NEXT: negb %cl
@@ -10359,35 +10359,35 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: pushq %rax
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups 32(%rdi), %ymm1
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%rsi), %r9d
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%rsi), %r8d
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (,%r9,8), %eax
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (,%r8,8), %eax
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %eax
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %r9d
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -128(%rsp,%r9), %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -120(%rsp,%r9), %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %r8d
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -128(%rsp,%r8), %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -120(%rsp,%r8), %r9
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r10
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %esi
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: notb %sil
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%r8,%r8), %rdi
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%r9,%r9), %rdi
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %rdi
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r10, %rdi
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -104(%rsp,%r9), %r10
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -104(%rsp,%r8), %r10
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r10, %rbx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -96(%rsp,%r9), %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -96(%rsp,%r8), %r12
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%r12,%r12), %r11
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r11
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %rbx, %r11
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -112(%rsp,%r9), %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -112(%rsp,%r8), %rbx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rbx, %r14
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r14
@@ -10395,11 +10395,11 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r10
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r14, %r10
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -88(%rsp,%r9), %r14
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -88(%rsp,%r8), %r14
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r14, %r13
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r13
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -80(%rsp,%r9), %rbp
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -80(%rsp,%r8), %rbp
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%rbp,%rbp), %r15
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r15
@@ -10412,20 +10412,20 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r12, %r14
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %rbp
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -72(%rsp,%r9), %r9
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%r9,%r9), %r12
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq -72(%rsp,%r8), %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leaq (%r8,%r8), %r12
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %r12
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %rbp, %r12
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r9
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: addq %rbx, %rbx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %esi, %ecx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shlq %cl, %rbx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r8, %rbx
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: orq %r9, %rbx
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r9
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r9, 56(%rdx)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: shrq %cl, %r8
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r8, 56(%rdx)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %rbx, 8(%rdx)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r12, 48(%rdx)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movq %r14, 32(%rdx)
@@ -10503,55 +10503,55 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: pushq %rbx
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups 32(%rdi), %ymm1
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%rsi), %esi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%rsi), %eax
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%rsi,8), %eax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%rax,8), %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %ecx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ecx, %esi
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %eax
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %ecx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %esi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, -128(%rsp,%rsi), %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: notb %al
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -120(%rsp,%rsi), %r10
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -112(%rsp,%rsi), %r9
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, -128(%rsp,%rax), %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: notb %cl
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -120(%rsp,%rax), %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -112(%rsp,%rax), %r9
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%r10,%r10), %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %rdi, %rdi
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %rdi, %rdi
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r8, %rdi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -104(%rsp,%rsi), %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %r11, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -96(%rsp,%rsi), %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -104(%rsp,%rax), %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %r11, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -96(%rsp,%rax), %r14
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%r14,%r14), %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %r8, %r8
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r8, %r8
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %rbx, %r8
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %r9, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %r9, %rbx
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addq %r11, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %r11, %r11
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r11, %r11
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %rbx, %r11
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -88(%rsp,%rsi), %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %rbx, %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -80(%rsp,%rsi), %r12
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -88(%rsp,%rax), %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %rbx, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -80(%rsp,%rax), %r12
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%r12,%r12), %r13
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %r13, %r13
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r13, %r13
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r15, %r13
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %r14, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %r14, %r14
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addq %rbx, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %rbx, %rbx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %rbx, %rbx
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r14, %rbx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %r12, %r14
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -72(%rsp,%rsi), %rsi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%rsi,%rsi), %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %r15, %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %r12, %r14
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq -72(%rsp,%rax), %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leaq (%rax,%rax), %r15
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r15, %r15
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r14, %r15
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %r10, %r10
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %r10, %r10
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: addq %r9, %r9
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rax, %r9, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r10, %rax
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rcx, %rsi, %rcx
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rcx, 56(%rdx)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rax, 8(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shlxq %rcx, %r9, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: orq %r10, %rcx
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: shrxq %rsi, %rax, %rax
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rax, 56(%rdx)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rcx, 8(%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r15, 48(%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %rbx, 32(%rdx)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movq %r13, 40(%rdx)
@@ -12043,31 +12043,30 @@ define void @lshr_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl {{[0-9]+}}(%esp), %ecx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%ecx), %ymm0
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups 32(%ecx), %ymm1
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%eax), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%eax), %eax
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, %esi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %esi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $60, %esi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 68(%esp,%esi), %edx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll $3, %ecx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $24, %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll $3, %eax
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $24, %eax
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, %edi
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %ecx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 72(%esp,%esi), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%eax,%eax), %ebx
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, %ebp
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %cl, %ch
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 72(%esp,%esi), %ecx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (%ecx,%ecx), %ebx
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %ch
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: notb %ch
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %ch, %cl
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shll %cl, %ebx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: orl %edi, %ebx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl 64(%esp,%esi), %edi
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %ebp, %eax
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movb %al, %cl
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: shrl %cl, %edi
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: addl %edx, %edx
@@ -13640,11 +13639,11 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups 32(%rdi), %ymm1
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%rsi), %ecx
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: leal (,%rcx,8), %eax
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %eax
; X64-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %ecx
@@ -13735,11 +13734,11 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups 32(%rdi), %ymm1
; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: movl (%rsi), %eax
-; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: leal (,%rax,8), %ecx
; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %ecx
; X64-HAVE-SHLD-NO-BMI2-AVX1-NEXT: andl $56, %eax
@@ -13789,11 +13788,11 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups 32(%rdi), %ymm1
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%rsi), %esi
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%rsi,8), %eax
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %eax
; X64-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %eax, %ecx
@@ -13861,11 +13860,11 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups 32(%rdi), %ymm1
; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%rsi), %eax
-; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%rax,8), %ecx
; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %ecx
; X64-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: andl $56, %eax
@@ -15384,11 +15383,11 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%ecx), %ymm0
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups 32(%ecx), %ymm1
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl (%eax), %eax
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %eax, %edx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: andl $60, %edx
; X86-NO-SHLD-NO-BMI2-AVX1-NEXT: movl %edx, {{[-0-9]+}}(%e{{[sb]}}p) # 4-byte Spill
@@ -15597,11 +15596,11 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups (%ecx), %ymm0
; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups 32(%ecx), %ymm1
; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: movl (%eax), %ecx
-; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: movl %ecx, %ebp
; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: andl $60, %ebp
; X86-HAVE-SHLD-NO-BMI2-AVX1-NEXT: leal {{[0-9]+}}(%esp), %eax
@@ -15707,11 +15706,11 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups (%ecx), %ymm0
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups 32(%ecx), %ymm1
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%eax), %eax
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
-; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%eax,8), %ebx
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: andl $24, %ebx
; X86-NO-SHLD-HAVE-BMI2-AVX1-NEXT: movl %ebx, %ecx
@@ -15876,11 +15875,11 @@ define void @shl_64bytes(ptr %src.ptr, ptr %byteOff.ptr, ptr %dst) nounwind {
; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups (%ecx), %ymm0
; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups 32(%ecx), %ymm1
; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: movl (%eax), %ebx
-; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
-; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: leal (,%ebx,8), %ecx
; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: andl $24, %ecx
; X86-HAVE-SHLD-HAVE-BMI2-AVX1-NEXT: andl $60, %ebx
@@ -16126,11 +16125,11 @@ define void @shl_64bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) nou
; X64-AVX1-NEXT: vmovups (%rdi), %ymm0
; X64-AVX1-NEXT: vmovups 32(%rdi), %ymm1
; X64-AVX1-NEXT: movl (%rsi), %eax
-; X64-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; X64-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
-; X64-AVX1-NEXT: vmovups %ymm2, -{{[0-9]+}}(%rsp)
; X64-AVX1-NEXT: vmovups %ymm1, -{{[0-9]+}}(%rsp)
; X64-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
+; X64-AVX1-NEXT: vmovups %ymm0, -{{[0-9]+}}(%rsp)
; X64-AVX1-NEXT: shll $3, %eax
; X64-AVX1-NEXT: andl $56, %eax
; X64-AVX1-NEXT: negl %eax
@@ -16346,11 +16345,11 @@ define void @shl_64bytes_qwordOff(ptr %src.ptr, ptr %qwordOff.ptr, ptr %dst) nou
; X86-AVX1-NEXT: vmovups (%edx), %ymm0
; X86-AVX1-NEXT: vmovups 32(%edx), %ymm1
; X86-AVX1-NEXT: movl (%ecx), %ecx
-; X86-AVX1-NEXT: vxorps %xmm2, %xmm2, %xmm2
-; X86-AVX1-NEXT: vmovups %ymm2, {{[0-9]+}}(%esp)
-; X86-AVX1-NEXT: vmovups %ymm2, (%esp)
; X86-AVX1-NEXT: vmovups %ymm1, {{[0-9]+}}(%esp)
; X86-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovups %ymm0, {{[0-9]+}}(%esp)
+; X86-AVX1-NEXT: vmovups %ymm0, (%esp)
; X86-AVX1-NEXT: shll $3, %ecx
; X86-AVX1-NEXT: andl $56, %ecx
; X86-AVX1-NEXT: leal {{[0-9]+}}(%esp), %edx
diff --git a/llvm/test/CodeGen/X86/zero-call-used-regs-simd.ll b/llvm/test/CodeGen/X86/zero-call-used-regs-simd.ll
index d9253e0ca127b..9bb218781290b 100644
--- a/llvm/test/CodeGen/X86/zero-call-used-regs-simd.ll
+++ b/llvm/test/CodeGen/X86/zero-call-used-regs-simd.ll
@@ -169,9 +169,6 @@ define void @zero_k(<8 x i32> %arg, <8 x i1> %mask) #0 {
; AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
; AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1
; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX1-NEXT: vzeroupper
; AVX1-NEXT: retq
;
@@ -182,7 +179,6 @@ define void @zero_k(<8 x i32> %arg, <8 x i1> %mask) #0 {
; AVX2-NEXT: vpmaskmovd %ymm0, %ymm1, 0
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
@@ -194,7 +190,6 @@ define void @zero_k(<8 x i32> %arg, <8 x i1> %mask) #0 {
; AVX512VL-NEXT: vmovdqa32 %ymm0, 0 {%k1}
; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: kxorw %k0, %k0, %k1
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll
index 5edea6d0fd68a..5e6515b75618a 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll
@@ -2079,9 +2079,9 @@ define void @vec256_v16i16_to_v2i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0
; AVX512BW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,6,7,17,9,10,11,12,13,14,15]
-; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-NEXT: vpermt2w %ymm0, %ymm1, %ymm2
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [16,1,2,3,4,5,6,7,17,9,10,11,12,13,14,15]
+; AVX512BW-NEXT: vpermi2w %ymm0, %ymm1, %ymm2
; AVX512BW-NEXT: vpaddb (%rdx), %zmm2, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rcx)
; AVX512BW-NEXT: vzeroupper
@@ -3977,11 +3977,10 @@ define void @vec384_v24i16_to_v8i48_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bi
; AVX512BW-SLOW: # %bb.0:
; AVX512BW-SLOW-NEXT: vmovdqa (%rdi), %ymm0
; AVX512BW-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512BW-SLOW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [16,1,2,17,4,5,18,7,8,19,10,11,20,13,14,21]
-; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-SLOW-NEXT: vpermt2w %ymm0, %ymm1, %ymm2
-; AVX512BW-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX512BW-SLOW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-SLOW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [16,1,2,17,4,5,18,7,8,19,10,11,20,13,14,21]
+; AVX512BW-SLOW-NEXT: vpermi2w %ymm0, %ymm1, %ymm2
+; AVX512BW-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4],xmm0[5],xmm1[6,7]
; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -3993,9 +3992,9 @@ define void @vec384_v24i16_to_v8i48_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bi
; AVX512BW-FAST: # %bb.0:
; AVX512BW-FAST-NEXT: vmovdqa (%rdi), %ymm0
; AVX512BW-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512BW-FAST-NEXT: vpmovsxbw {{.*#+}} ymm1 = [16,1,2,17,4,5,18,7,8,19,10,11,20,13,14,21]
-; AVX512BW-FAST-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-FAST-NEXT: vpermt2w %ymm0, %ymm1, %ymm2
+; AVX512BW-FAST-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-FAST-NEXT: vpmovsxbw {{.*#+}} ymm2 = [16,1,2,17,4,5,18,7,8,19,10,11,20,13,14,21]
+; AVX512BW-FAST-NEXT: vpermi2w %ymm0, %ymm1, %ymm2
; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[12,13],zero,zero,zero,zero,xmm0[14,15],zero,zero,zero,zero
; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm0
; AVX512BW-FAST-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4291,11 +4290,10 @@ define void @vec384_v24i16_to_v4i96_factor6(ptr %in.vec.base.ptr, ptr %in.vec.bi
; AVX512BW-SLOW: # %bb.0:
; AVX512BW-SLOW-NEXT: vmovdqa (%rdi), %ymm0
; AVX512BW-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512BW-SLOW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,17,7,8,9,10,11,18,13,14,15]
-; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-SLOW-NEXT: vpermt2w %ymm0, %ymm1, %ymm2
-; AVX512BW-SLOW-NEXT: vpsrld $16, %xmm0, %xmm0
; AVX512BW-SLOW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-SLOW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [16,1,2,3,4,5,17,7,8,9,10,11,18,13,14,15]
+; AVX512BW-SLOW-NEXT: vpermi2w %ymm0, %ymm1, %ymm2
+; AVX512BW-SLOW-NEXT: vpsrld $16, %xmm0, %xmm0
; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1],xmm0[2],xmm1[3,4,5,6,7]
; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4307,9 +4305,9 @@ define void @vec384_v24i16_to_v4i96_factor6(ptr %in.vec.base.ptr, ptr %in.vec.bi
; AVX512BW-FAST: # %bb.0:
; AVX512BW-FAST-NEXT: vmovdqa (%rdi), %ymm0
; AVX512BW-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512BW-FAST-NEXT: vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,17,7,8,9,10,11,18,13,14,15]
-; AVX512BW-FAST-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-FAST-NEXT: vpermt2w %ymm0, %ymm1, %ymm2
+; AVX512BW-FAST-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-FAST-NEXT: vpmovsxbw {{.*#+}} ymm2 = [16,1,2,3,4,5,17,7,8,9,10,11,18,13,14,15]
+; AVX512BW-FAST-NEXT: vpermi2w %ymm0, %ymm1, %ymm2
; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[6,7],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm0
; AVX512BW-FAST-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4443,7 +4441,6 @@ define void @vec384_v24i16_to_v3i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b
; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2,3,4,5,6,7]
; AVX512F-SLOW-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero
; AVX512F-SLOW-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,1,1,3]
-; AVX512F-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm2[1,2,3,4,5,6,7],ymm0[8],ymm2[9,10,11,12,13,14,15]
; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm0, %ymm0
; AVX512F-SLOW-NEXT: vpaddb 32(%rdx), %ymm1, %ymm1
@@ -4472,11 +4469,10 @@ define void @vec384_v24i16_to_v3i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b
; AVX512BW-SLOW: # %bb.0:
; AVX512BW-SLOW-NEXT: vmovdqa (%rdi), %ymm0
; AVX512BW-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512BW-SLOW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,6,7,17,9,10,11,12,13,14,15]
-; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-SLOW-NEXT: vpermt2w %ymm0, %ymm1, %ymm2
-; AVX512BW-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; AVX512BW-SLOW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-SLOW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [16,1,2,3,4,5,6,7,17,9,10,11,12,13,14,15]
+; AVX512BW-SLOW-NEXT: vpermi2w %ymm0, %ymm1, %ymm2
+; AVX512BW-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4488,9 +4484,9 @@ define void @vec384_v24i16_to_v3i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b
; AVX512BW-FAST: # %bb.0:
; AVX512BW-FAST-NEXT: vmovdqa (%rdi), %ymm0
; AVX512BW-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512BW-FAST-NEXT: vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,6,7,17,9,10,11,12,13,14,15]
-; AVX512BW-FAST-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-FAST-NEXT: vpermt2w %ymm0, %ymm1, %ymm2
+; AVX512BW-FAST-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-FAST-NEXT: vpmovsxbw {{.*#+}} ymm2 = [16,1,2,3,4,5,6,7,17,9,10,11,12,13,14,15]
+; AVX512BW-FAST-NEXT: vpermi2w %ymm0, %ymm1, %ymm2
; AVX512BW-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[4,5],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX512BW-FAST-NEXT: vinserti32x4 $2, %xmm0, %zmm2, %zmm0
; AVX512BW-FAST-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4595,9 +4591,9 @@ define void @vec384_v24i16_to_v2i192_factor12(ptr %in.vec.base.ptr, ptr %in.vec.
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa (%rdi), %ymm0
; AVX512BW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm1 = [16,1,2,3,4,5,6,7,8,9,10,11,17,13,14,15]
-; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-NEXT: vpermt2w %ymm0, %ymm1, %ymm2
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} ymm2 = [16,1,2,3,4,5,6,7,8,9,10,11,17,13,14,15]
+; AVX512BW-NEXT: vpermi2w %ymm0, %ymm1, %ymm2
; AVX512BW-NEXT: vpaddb (%rdx), %zmm2, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rcx)
; AVX512BW-NEXT: vzeroupper
@@ -4916,12 +4912,11 @@ define void @vec384_v12i32_to_v4i96_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bi
; AVX2-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm2 = ymm0[0,0,2,1]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0],ymm1[1,2],ymm2[3],ymm1[4,5],ymm2[6],ymm1[7]
; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3]
; AVX2-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
-; AVX2-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpaddb (%rdx), %ymm2, %ymm1
; AVX2-SLOW-NEXT: vmovdqa %ymm1, (%rcx)
; AVX2-SLOW-NEXT: vmovdqa %ymm0, 32(%rcx)
; AVX2-SLOW-NEXT: vzeroupper
@@ -4961,9 +4956,9 @@ define void @vec384_v12i32_to_v4i96_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bi
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,2,17,4,5,18,7,8,19,10,11,0,0,0,0]
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpermt2d %zmm0, %zmm1, %zmm2
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm2 = [16,1,2,17,4,5,18,7,8,19,10,11,0,0,0,0]
+; AVX512F-NEXT: vpermi2d %zmm0, %zmm1, %zmm2
; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm0
; AVX512F-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm1
@@ -5078,8 +5073,7 @@ define void @vec384_v12i32_to_v3i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.b
; AVX2-SLOW-NEXT: vmovdqa (%rdi), %xmm1
; AVX2-SLOW-NEXT: vpaddb (%rsi), %xmm1, %xmm1
; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; AVX2-SLOW-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0],xmm3[1,2,3]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm2 = xmm2[0],xmm0[1,2,3]
; AVX2-SLOW-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,1,1,3]
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1,2,3],ymm1[4],ymm0[5,6,7]
@@ -5126,9 +5120,9 @@ define void @vec384_v12i32_to_v3i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.b
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,2,3,17,5,6,7,18,9,10,11,0,0,0,0]
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpermt2d %zmm0, %zmm1, %zmm2
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm2 = [16,1,2,3,17,5,6,7,18,9,10,11,0,0,0,0]
+; AVX512F-NEXT: vpermi2d %zmm0, %zmm1, %zmm2
; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm0
; AVX512F-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm1
@@ -6872,9 +6866,9 @@ define void @vec512_v32i16_to_v4i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vmovdqa64 (%rdi), %zmm0
; AVX512BW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
-; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [32,1,2,3,4,5,6,7,33,9,10,11,12,13,14,15,34,1,2,3,4,5,6,7,35,9,10,11,12,13,14,15]
-; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-NEXT: vpermt2w %zmm0, %zmm1, %zmm2
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm2 = [32,1,2,3,4,5,6,7,33,9,10,11,12,13,14,15,34,1,2,3,4,5,6,7,35,9,10,11,12,13,14,15]
+; AVX512BW-NEXT: vpermi2w %zmm0, %zmm1, %zmm2
; AVX512BW-NEXT: vpaddb (%rdx), %zmm2, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rcx)
; AVX512BW-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
index a28c23bb8a61e..6489f950482b6 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
@@ -3775,9 +3775,8 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14],ymm0[15]
-; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14],ymm0[15]
+; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
; AVX2-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX2-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -3795,9 +3794,8 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
; AVX512F-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
; AVX512F-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14],ymm0[15]
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14],ymm0[15]
+; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512F-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -3834,9 +3832,8 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
; AVX512DQ-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7]
; AVX512DQ-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14],ymm0[15]
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0],ymm0[1],ymm2[2,3],ymm0[4],ymm2[5,6],ymm0[7],ymm2[8],ymm0[9],ymm2[10,11],ymm0[12],ymm2[13,14],ymm0[15]
+; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512DQ-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -3872,7 +3869,6 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpermi2w %zmm2, %zmm0, %zmm1
; AVX512BW-SLOW-NEXT: vpbroadcastw %xmm0, %xmm0
-; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4093,7 +4089,6 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpermi2w %zmm2, %zmm0, %zmm1
; AVX512BW-SLOW-NEXT: vpbroadcastw %xmm0, %xmm0
-; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7]
; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4200,9 +4195,8 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
; AVX2-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5],xmm0[6],xmm1[7]
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15]
-; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
; AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
; AVX2-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX2-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -4220,9 +4214,8 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
; AVX512F-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5],xmm0[6],xmm1[7]
; AVX512F-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15]
-; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512F-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX512F-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512F-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -4259,9 +4252,8 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
; AVX512DQ-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5],xmm0[6],xmm1[7]
; AVX512DQ-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15]
-; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5,6,7]
-; AVX512DQ-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm3 = ymm2[0,1,2,3],ymm0[4],ymm2[5,6,7,8,9,10,11],ymm0[12],ymm2[13,14,15]
+; AVX512DQ-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm3[4,5,6,7]
; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX512DQ-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -4297,7 +4289,6 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpermi2w %zmm2, %zmm0, %zmm1
; AVX512BW-SLOW-NEXT: vpbroadcastw %xmm0, %xmm0
-; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4446,7 +4437,6 @@ define void @vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3(ptr %i
; AVX512BW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-NEXT: vpermi2w %zmm2, %zmm0, %zmm1
-; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7]
; AVX512BW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4676,7 +4666,6 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4,5,6,7]
; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6],ymm2[7]
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
; AVX2-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -4766,7 +4755,6 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
; AVX512BW-SLOW-NEXT: vpbroadcastd %xmm0, %xmm0
; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1],xmm0[2],xmm2[3]
-; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5],ymm1[6],ymm2[7]
; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -4966,7 +4954,6 @@ define void @vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4(ptr %in.
; AVX512BW-SLOW-NEXT: vpbroadcastd %xmm0, %xmm0
; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3]
-; AVX512BW-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm2[4,5],ymm1[6],ymm2[7]
; AVX512BW-SLOW-NEXT: vinserti32x4 $2, %xmm0, %zmm1, %zmm0
; AVX512BW-SLOW-NEXT: vpaddb (%rdx), %zmm0, %zmm0
@@ -5066,7 +5053,6 @@ define void @vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3(ptr %i
; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0],ymm1[1,2,3],ymm2[4,5,6,7]
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4],ymm2[5,6,7]
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3]
; AVX2-NEXT: vpaddb (%rdx), %ymm1, %ymm1
; AVX2-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
@@ -5215,9 +5201,9 @@ define void @vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2(ptr %i
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,30,31,4,5,16,7,0,0,0,0,0,0,0,0]
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpermt2d %zmm0, %zmm1, %zmm2
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm2 = [16,29,30,31,4,5,16,7,0,0,0,0,0,0,0,0]
+; AVX512F-NEXT: vpermi2d %zmm0, %zmm1, %zmm2
; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm0
; AVX512F-NEXT: vmovaps 32(%rdx), %ymm1
; AVX512F-NEXT: vmovaps %ymm1, 32(%rcx)
@@ -5232,9 +5218,9 @@ define void @vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2(ptr %i
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,30,31,4,5,16,7,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpermt2d %zmm0, %zmm1, %zmm2
+; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm2 = [16,29,30,31,4,5,16,7,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT: vpermi2d %zmm0, %zmm1, %zmm2
; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm0
; AVX512DQ-NEXT: vmovaps 32(%rdx), %ymm1
; AVX512DQ-NEXT: vmovaps %ymm1, 32(%rcx)
@@ -6792,9 +6778,9 @@ define void @vec512_i32_widen_to_i128_factor4_broadcast_to_v4i128_factor4(ptr %i
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,2,3,16,5,6,7,16,9,10,11,16,13,14,15]
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpermt2d %zmm0, %zmm1, %zmm2
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm2 = [16,1,2,3,16,5,6,7,16,9,10,11,16,13,14,15]
+; AVX512F-NEXT: vpermi2d %zmm0, %zmm1, %zmm2
; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm0
; AVX512F-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm1
@@ -6807,9 +6793,9 @@ define void @vec512_i32_widen_to_i128_factor4_broadcast_to_v4i128_factor4(ptr %i
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,2,3,16,5,6,7,16,9,10,11,16,13,14,15]
-; AVX512DQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpermt2d %zmm0, %zmm1, %zmm2
+; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm2 = [16,1,2,3,16,5,6,7,16,9,10,11,16,13,14,15]
+; AVX512DQ-NEXT: vpermi2d %zmm0, %zmm1, %zmm2
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm2, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm1
@@ -6909,9 +6895,9 @@ define void @vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2(ptr %i
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,2,3,4,5,6,7,16,9,10,11,12,13,14,15]
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpermt2d %zmm0, %zmm1, %zmm2
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm2 = [16,1,2,3,4,5,6,7,16,9,10,11,12,13,14,15]
+; AVX512F-NEXT: vpermi2d %zmm0, %zmm1, %zmm2
; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm0
; AVX512F-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm1
@@ -6924,9 +6910,9 @@ define void @vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2(ptr %i
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,2,3,4,5,6,7,16,9,10,11,12,13,14,15]
-; AVX512DQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpermt2d %zmm0, %zmm1, %zmm2
+; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm2 = [16,1,2,3,4,5,6,7,16,9,10,11,12,13,14,15]
+; AVX512DQ-NEXT: vpermi2d %zmm0, %zmm1, %zmm2
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm2, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm1
@@ -7111,9 +7097,9 @@ define void @vec512_i64_widen_to_i256_factor4_broadcast_to_v2i256_factor2(ptr %i
; AVX512F: # %bb.0:
; AVX512F-NEXT: vmovdqa (%rdi), %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,1,2,3,8,5,6,7]
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm2 = [8,1,2,3,8,5,6,7]
+; AVX512F-NEXT: vpermi2q %zmm0, %zmm1, %zmm2
; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm0
; AVX512F-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rdx), %ymm2, %ymm1
@@ -7126,9 +7112,9 @@ define void @vec512_i64_widen_to_i256_factor4_broadcast_to_v2i256_factor2(ptr %i
; AVX512DQ: # %bb.0:
; AVX512DQ-NEXT: vmovdqa (%rdi), %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,1,2,3,8,5,6,7]
-; AVX512DQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpermt2q %zmm0, %zmm1, %zmm2
+; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm2 = [8,1,2,3,8,5,6,7]
+; AVX512DQ-NEXT: vpermi2q %zmm0, %zmm1, %zmm2
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm2, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rdx), %ymm2, %ymm1
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
index 21a3df0456de1..cbd4e9e0e614e 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast_from_memory.ll
@@ -2922,9 +2922,9 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
;
; AVX512BW-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [32,57,32,59,32,61,32,63,32,9,32,11,32,13,32,15,32,17,32,19,32,21,32,23,0,0,0,0,0,0,0,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2w (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [32,57,32,59,32,61,32,63,32,9,32,11,32,13,32,15,32,17,32,19,32,21,32,23,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT: vpermi2w (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -2997,14 +2997,13 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
;
; AVX2-SLOW-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8:
; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14],ymm0[15]
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],mem[1,2],xmm0[3],mem[4,5],xmm0[6],mem[7]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX2-SLOW-NEXT: vpaddb (%rsi), %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7],ymm0[8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14],ymm1[15]
+; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],mem[1,2],xmm1[3],mem[4,5],xmm1[6],mem[7]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
+; AVX2-SLOW-NEXT: vpaddb (%rsi), %ymm2, %ymm1
; AVX2-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX2-SLOW-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX2-SLOW-NEXT: vmovdqa %ymm1, (%rdx)
@@ -3013,11 +3012,11 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
;
; AVX2-FAST-PERLANE-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8:
; AVX2-FAST-PERLANE: # %bb.0:
-; AVX2-FAST-PERLANE-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX2-FAST-PERLANE-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14],ymm0[15]
-; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1,2],xmm0[3],mem[4,5],xmm0[6],mem[7]
-; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-FAST-PERLANE-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7],ymm0[8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14],ymm1[15]
+; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],mem[1,2],xmm1[3],mem[4,5],xmm1[6],mem[7]
+; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %xmm1
; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm1[0,1],zero,zero,zero,zero,xmm1[0,1],zero,zero,zero,zero
; AVX2-FAST-PERLANE-NEXT: vpaddb (%rsi), %ymm0, %ymm0
@@ -3029,11 +3028,11 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
;
; AVX2-FAST-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8:
; AVX2-FAST: # %bb.0:
-; AVX2-FAST-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX2-FAST-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14],ymm0[15]
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1,2],xmm0[3],mem[4,5],xmm0[6],mem[7]
-; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7],ymm0[8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14],ymm1[15]
+; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],mem[1,2],xmm1[3],mem[4,5],xmm1[6],mem[7]
+; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm1
; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm1[0,1],zero,zero,zero,zero,xmm1[0,1],zero,zero,zero,zero
; AVX2-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
@@ -3045,14 +3044,13 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
;
; AVX512F-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14],ymm0[15]
-; AVX512F-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],mem[1,2],xmm0[3],mem[4,5],xmm0[6],mem[7]
-; AVX512F-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX512F-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7],ymm0[8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14],ymm1[15]
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],mem[1,2],xmm1[3],mem[4,5],xmm1[6],mem[7]
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
+; AVX512F-NEXT: vpaddb (%rsi), %ymm2, %ymm1
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rdx)
@@ -3061,14 +3059,13 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
;
; AVX512DQ-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0],ymm0[1],ymm1[2,3],ymm0[4],ymm1[5,6],ymm0[7],ymm1[8],ymm0[9],ymm1[10,11],ymm0[12],ymm1[13,14],ymm0[15]
-; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],mem[1,2],xmm0[3],mem[4,5],xmm0[6],mem[7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7]
-; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0],ymm1[1],ymm0[2,3],ymm1[4],ymm0[5,6],ymm1[7],ymm0[8],ymm1[9],ymm0[10,11],ymm1[12],ymm0[13,14],ymm1[15]
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],mem[1,2],xmm1[3],mem[4,5],xmm1[6],mem[7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4],xmm1[5],xmm0[6,7]
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm2, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
@@ -3077,9 +3074,9 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in.
;
; AVX512BW-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [32,57,58,32,60,61,32,63,8,32,10,11,32,13,14,32,16,17,32,19,20,32,22,23,0,0,0,0,0,0,0,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2w (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [32,57,58,32,60,61,32,63,8,32,10,11,32,13,14,32,16,17,32,19,20,32,22,23,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT: vpermi2w (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -3251,9 +3248,9 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
;
; AVX512BW-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [32,57,58,59,32,61,62,63,32,9,10,11,32,13,14,15,32,17,18,19,32,21,22,23,0,0,0,0,0,0,0,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2w (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [32,57,58,59,32,61,62,63,32,9,10,11,32,13,14,15,32,17,18,19,32,21,22,23,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT: vpermi2w (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -3329,14 +3326,13 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
;
; AVX2-SLOW-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4:
; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX2-SLOW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],mem[1,2,3,4,5],xmm0[6],mem[7]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
-; AVX2-SLOW-NEXT: vpaddb (%rsi), %ymm1, %ymm1
+; AVX2-SLOW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX2-SLOW-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7,8,9,10,11],ymm1[12],ymm0[13,14,15]
+; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],mem[1,2,3,4,5],xmm1[6],mem[7]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX2-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4,5,6,7]
+; AVX2-SLOW-NEXT: vpaddb (%rsi), %ymm2, %ymm1
; AVX2-SLOW-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX2-SLOW-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX2-SLOW-NEXT: vmovdqa %ymm1, (%rdx)
@@ -3345,11 +3341,11 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
;
; AVX2-FAST-PERLANE-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4:
; AVX2-FAST-PERLANE: # %bb.0:
-; AVX2-FAST-PERLANE-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX2-FAST-PERLANE-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
-; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1,2,3,4,5],xmm0[6],mem[7]
-; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-PERLANE-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-FAST-PERLANE-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7,8,9,10,11],ymm1[12],ymm0[13,14,15]
+; AVX2-FAST-PERLANE-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],mem[1,2,3,4,5],xmm1[6],mem[7]
+; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX2-FAST-PERLANE-NEXT: vmovdqa (%rdi), %xmm1
; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm1[0,1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX2-FAST-PERLANE-NEXT: vpaddb (%rsi), %ymm0, %ymm0
@@ -3361,11 +3357,11 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
;
; AVX2-FAST-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4:
; AVX2-FAST: # %bb.0:
-; AVX2-FAST-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX2-FAST-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
-; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],mem[1,2,3,4,5],xmm0[6],mem[7]
-; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; AVX2-FAST-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX2-FAST-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7,8,9,10,11],ymm1[12],ymm0[13,14,15]
+; AVX2-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],mem[1,2,3,4,5],xmm1[6],mem[7]
+; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; AVX2-FAST-NEXT: vmovdqa (%rdi), %xmm1
; AVX2-FAST-NEXT: vpshufb {{.*#+}} xmm1 = zero,zero,zero,zero,xmm1[0,1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; AVX2-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
@@ -3377,14 +3373,13 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
;
; AVX512F-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
-; AVX512F-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],mem[1,2,3,4,5],xmm0[6],mem[7]
-; AVX512F-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512F-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512F-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
-; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX512F-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7,8,9,10,11],ymm1[12],ymm0[13,14,15]
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],mem[1,2,3,4,5],xmm1[6],mem[7]
+; AVX512F-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512F-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4,5,6,7]
+; AVX512F-NEXT: vpaddb (%rsi), %ymm2, %ymm1
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512F-NEXT: vmovdqa %ymm1, (%rdx)
@@ -3393,14 +3388,13 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
;
; AVX512DQ-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
-; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],mem[1,2,3,4,5],xmm0[6],mem[7]
-; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm1 = ymm2[0,1,2,3],ymm1[4,5,6,7]
-; AVX512DQ-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4,5,6,7]
-; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm2 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7,8,9,10,11],ymm1[12],ymm0[13,14,15]
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm3 = xmm1[0],mem[1,2,3,4,5],xmm1[6],mem[7]
+; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm2 = ymm3[0,1,2,3],ymm2[4,5,6,7]
+; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2],xmm0[3,4,5,6,7]
+; AVX512DQ-NEXT: vpaddb (%rsi), %ymm2, %ymm1
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rdx)
; AVX512DQ-NEXT: vmovdqa %ymm1, (%rdx)
@@ -3409,9 +3403,9 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in.
;
; AVX512BW-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [32,57,58,59,60,61,32,63,8,9,10,11,32,13,14,15,16,17,32,19,20,21,22,23,0,0,0,0,0,0,0,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2w (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [32,57,58,59,60,61,32,63,8,9,10,11,32,13,14,15,16,17,32,19,20,21,22,23,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT: vpermi2w (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -3523,9 +3517,9 @@ define void @vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3(ptr %i
;
; AVX512BW-LABEL: vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [32,57,58,59,60,61,62,63,32,9,10,11,12,13,14,15,32,17,18,19,20,21,22,23,0,0,0,0,0,0,0,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2w (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [32,57,58,59,60,61,62,63,32,9,10,11,12,13,14,15,32,17,18,19,20,21,22,23,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT: vpermi2w (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -3593,9 +3587,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
;
; AVX2-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX2-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7,8,9,10,11],ymm1[12],ymm0[13,14,15]
; AVX2-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX2-NEXT: vpblendw {{.*#+}} xmm1 = mem[0],xmm1[1,2,3,4,5,6,7]
; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
@@ -3608,9 +3602,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
;
; AVX512F-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX512F-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7,8,9,10,11],ymm1[12],ymm0[13,14,15]
; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512F-NEXT: vpblendw {{.*#+}} xmm1 = mem[0],xmm1[1,2,3,4,5,6,7]
; AVX512F-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
@@ -3623,9 +3617,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
;
; AVX512DQ-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpbroadcastw (%rdi), %ymm0
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4],ymm1[5,6,7,8,9,10,11],ymm0[12],ymm1[13,14,15]
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpbroadcastw (%rdi), %ymm1
+; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4],ymm0[5,6,7,8,9,10,11],ymm1[12],ymm0[13,14,15]
; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512DQ-NEXT: vpblendw {{.*#+}} xmm1 = mem[0],xmm1[1,2,3,4,5,6,7]
; AVX512DQ-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
@@ -3638,9 +3632,9 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr %
;
; AVX512BW-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [32,57,58,59,60,61,62,63,8,9,10,11,32,13,14,15,16,17,18,19,20,21,22,23,0,0,0,0,0,0,0,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2w (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [32,57,58,59,60,61,62,63,8,9,10,11,32,13,14,15,16,17,18,19,20,21,22,23,0,0,0,0,0,0,0,0]
+; AVX512BW-NEXT: vpermi2w (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -3717,7 +3711,6 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4,5,6,7]
; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4],ymm2[5],ymm0[6],ymm2[7]
-; AVX2-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2],xmm2[3]
; AVX2-SLOW-NEXT: vpaddb 32(%rsi), %ymm1, %ymm1
; AVX2-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
@@ -3760,9 +3753,9 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
;
; AVX512F-LABEL: vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,29,16,31,16,5,16,7,16,9,16,11,0,0,0,0]
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,16,31,16,5,16,7,16,9,16,11,0,0,0,0]
+; AVX512F-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -3773,9 +3766,9 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
;
; AVX512DQ-LABEL: vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,29,16,31,16,5,16,7,16,9,16,11,0,0,0,0]
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,16,31,16,5,16,7,16,9,16,11,0,0,0,0]
+; AVX512DQ-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -3786,9 +3779,9 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in.
;
; AVX512BW-LABEL: vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,29,16,31,16,5,16,7,16,9,16,11,0,0,0,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,16,31,16,5,16,7,16,9,16,11,0,0,0,0]
+; AVX512BW-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -3910,9 +3903,9 @@ define void @vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4(ptr %in.
;
; AVX512F-LABEL: vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,29,30,16,4,5,16,7,8,16,10,11,0,0,0,0]
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,30,16,4,5,16,7,8,16,10,11,0,0,0,0]
+; AVX512F-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -3923,9 +3916,9 @@ define void @vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4(ptr %in.
;
; AVX512DQ-LABEL: vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,29,30,16,4,5,16,7,8,16,10,11,0,0,0,0]
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,30,16,4,5,16,7,8,16,10,11,0,0,0,0]
+; AVX512DQ-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -3936,9 +3929,9 @@ define void @vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4(ptr %in.
;
; AVX512BW-LABEL: vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,29,30,16,4,5,16,7,8,16,10,11,0,0,0,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,30,16,4,5,16,7,8,16,10,11,0,0,0,0]
+; AVX512BW-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -4019,9 +4012,9 @@ define void @vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3(ptr %i
;
; AVX512F-LABEL: vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,29,30,31,16,5,6,7,16,9,10,11,0,0,0,0]
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,30,31,16,5,6,7,16,9,10,11,0,0,0,0]
+; AVX512F-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -4032,9 +4025,9 @@ define void @vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3(ptr %i
;
; AVX512DQ-LABEL: vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,29,30,31,16,5,6,7,16,9,10,11,0,0,0,0]
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,30,31,16,5,6,7,16,9,10,11,0,0,0,0]
+; AVX512DQ-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -4045,9 +4038,9 @@ define void @vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3(ptr %i
;
; AVX512BW-LABEL: vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,29,30,31,16,5,6,7,16,9,10,11,0,0,0,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,30,31,16,5,6,7,16,9,10,11,0,0,0,0]
+; AVX512BW-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -4129,9 +4122,9 @@ define void @vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2(ptr %i
;
; AVX512F-LABEL: vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,29,30,31,4,5,16,7,0,0,0,0,0,0,0,0]
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,30,31,4,5,16,7,0,0,0,0,0,0,0,0]
+; AVX512F-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm0
; AVX512F-NEXT: vmovaps 32(%rsi), %ymm1
; AVX512F-NEXT: vmovaps %ymm1, 32(%rdx)
@@ -4141,9 +4134,9 @@ define void @vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2(ptr %i
;
; AVX512DQ-LABEL: vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,29,30,31,4,5,16,7,0,0,0,0,0,0,0,0]
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,30,31,4,5,16,7,0,0,0,0,0,0,0,0]
+; AVX512DQ-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm0
; AVX512DQ-NEXT: vmovaps 32(%rsi), %ymm1
; AVX512DQ-NEXT: vmovaps %ymm1, 32(%rdx)
@@ -4153,9 +4146,9 @@ define void @vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2(ptr %i
;
; AVX512BW-LABEL: vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,29,30,31,4,5,16,7,8,9,10,11,0,0,0,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,29,30,31,4,5,16,7,8,9,10,11,0,0,0,0]
+; AVX512BW-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -4233,9 +4226,9 @@ define void @vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3(ptr %i
;
; AVX512F-LABEL: vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,15,8,3,8,5,0,0]
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,15,8,3,8,5,0,0]
+; AVX512F-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -4246,9 +4239,9 @@ define void @vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3(ptr %i
;
; AVX512DQ-LABEL: vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,15,8,3,8,5,0,0]
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,15,8,3,8,5,0,0]
+; AVX512DQ-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -4259,9 +4252,9 @@ define void @vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3(ptr %i
;
; AVX512BW-LABEL: vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,15,8,3,8,5,0,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,15,8,3,8,5,0,0]
+; AVX512BW-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -4336,9 +4329,9 @@ define void @vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2(ptr %i
;
; AVX512F-LABEL: vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,15,2,8,0,0,0,0]
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,15,2,8,0,0,0,0]
+; AVX512F-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm0
; AVX512F-NEXT: vmovaps 32(%rsi), %ymm1
; AVX512F-NEXT: vmovaps %ymm1, 32(%rdx)
@@ -4348,9 +4341,9 @@ define void @vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2(ptr %i
;
; AVX512DQ-LABEL: vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,15,2,8,0,0,0,0]
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,15,2,8,0,0,0,0]
+; AVX512DQ-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm0
; AVX512DQ-NEXT: vmovaps 32(%rsi), %ymm1
; AVX512DQ-NEXT: vmovaps %ymm1, 32(%rdx)
@@ -4360,9 +4353,9 @@ define void @vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2(ptr %i
;
; AVX512BW-LABEL: vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,15,2,8,4,5,0,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,15,2,8,4,5,0,0]
+; AVX512BW-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -4825,9 +4818,9 @@ define void @vec512_i8_widen_to_i256_factor32_broadcast_to_v2i256_factor2(ptr %i
;
; AVX512BW-LABEL: vec512_i8_widen_to_i256_factor32_broadcast_to_v2i256_factor2:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,0,2,0,8,0,6,0]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,0,2,0,8,0,6,0]
+; AVX512BW-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm1, %zmm0
; AVX512BW-NEXT: vpaddb (%rsi), %zmm0, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
@@ -5274,9 +5267,9 @@ define void @vec512_i16_widen_to_i256_factor16_broadcast_to_v2i256_factor2(ptr %
;
; AVX512BW-LABEL: vec512_i16_widen_to_i256_factor16_broadcast_to_v2i256_factor2:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm0 = [32,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,32,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2w (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbw {{.*#+}} zmm1 = [32,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,32,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31]
+; AVX512BW-NEXT: vpermi2w (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -5330,9 +5323,9 @@ define void @vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8(ptr %in.
;
; AVX1-LABEL: vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8:
; AVX1: # %bb.0:
-; AVX1-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[1,3],ymm0[4,4],ymm1[5,7]
+; AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm1[0,0],ymm0[1,3],ymm1[4,4],ymm0[5,7]
; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm2
@@ -5348,9 +5341,9 @@ define void @vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8(ptr %in.
;
; AVX2-LABEL: vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastd (%rdi), %ymm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7]
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vpbroadcastd (%rdi), %ymm1
+; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[3],ymm1[4],ymm0[5],ymm1[6],ymm0[7]
; AVX2-NEXT: vpaddb 32(%rsi), %ymm0, %ymm1
; AVX2-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
@@ -5360,9 +5353,9 @@ define void @vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8(ptr %in.
;
; AVX512F-LABEL: vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,1,16,3,16,5,16,7,16,9,16,11,16,13,16,15]
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,16,3,16,5,16,7,16,9,16,11,16,13,16,15]
+; AVX512F-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -5373,9 +5366,9 @@ define void @vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8(ptr %in.
;
; AVX512DQ-LABEL: vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,1,16,3,16,5,16,7,16,9,16,11,16,13,16,15]
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,16,3,16,5,16,7,16,9,16,11,16,13,16,15]
+; AVX512DQ-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -5386,9 +5379,9 @@ define void @vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8(ptr %in.
;
; AVX512BW-LABEL: vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,1,16,3,16,5,16,7,16,9,16,11,16,13,16,15]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,16,3,16,5,16,7,16,9,16,11,16,13,16,15]
+; AVX512BW-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -5451,9 +5444,9 @@ define void @vec512_i32_widen_to_i128_factor4_broadcast_to_v4i128_factor4(ptr %i
;
; AVX512F-LABEL: vec512_i32_widen_to_i128_factor4_broadcast_to_v4i128_factor4:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,1,2,3,16,5,6,7,16,9,10,11,16,13,14,15]
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,2,3,16,5,6,7,16,9,10,11,16,13,14,15]
+; AVX512F-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -5464,9 +5457,9 @@ define void @vec512_i32_widen_to_i128_factor4_broadcast_to_v4i128_factor4(ptr %i
;
; AVX512DQ-LABEL: vec512_i32_widen_to_i128_factor4_broadcast_to_v4i128_factor4:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,1,2,3,16,5,6,7,16,9,10,11,16,13,14,15]
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,2,3,16,5,6,7,16,9,10,11,16,13,14,15]
+; AVX512DQ-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -5477,9 +5470,9 @@ define void @vec512_i32_widen_to_i128_factor4_broadcast_to_v4i128_factor4(ptr %i
;
; AVX512BW-LABEL: vec512_i32_widen_to_i128_factor4_broadcast_to_v4i128_factor4:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,1,2,3,16,5,6,7,16,9,10,11,16,13,14,15]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,2,3,16,5,6,7,16,9,10,11,16,13,14,15]
+; AVX512BW-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -5534,9 +5527,9 @@ define void @vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2(ptr %i
;
; AVX512F-LABEL: vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,1,2,3,4,5,6,7,16,9,10,11,12,13,14,15]
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,2,3,4,5,6,7,16,9,10,11,12,13,14,15]
+; AVX512F-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -5547,9 +5540,9 @@ define void @vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2(ptr %i
;
; AVX512DQ-LABEL: vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,1,2,3,4,5,6,7,16,9,10,11,12,13,14,15]
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,2,3,4,5,6,7,16,9,10,11,12,13,14,15]
+; AVX512DQ-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -5560,9 +5553,9 @@ define void @vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2(ptr %i
;
; AVX512BW-LABEL: vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm0 = [16,1,2,3,4,5,6,7,16,9,10,11,12,13,14,15]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2d (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbd {{.*#+}} zmm1 = [16,1,2,3,4,5,6,7,16,9,10,11,12,13,14,15]
+; AVX512BW-NEXT: vpermi2d (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -5596,9 +5589,9 @@ define void @vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4(ptr %i
;
; AVX1-LABEL: vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4:
; AVX1: # %bb.0:
-; AVX1-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1]
-; AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX1-NEXT: vxorps %xmm0, %xmm0, %xmm0
+; AVX1-NEXT: vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm2
; AVX1-NEXT: vpaddb 32(%rsi), %xmm0, %xmm3
@@ -5613,9 +5606,9 @@ define void @vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4(ptr %i
;
; AVX2-LABEL: vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastq (%rdi), %ymm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7]
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: vpbroadcastq (%rdi), %ymm1
+; AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1],ymm0[2,3],ymm1[4,5],ymm0[6,7]
; AVX2-NEXT: vpaddb 32(%rsi), %ymm0, %ymm1
; AVX2-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX2-NEXT: vmovdqa %ymm0, (%rdx)
@@ -5625,9 +5618,9 @@ define void @vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4(ptr %i
;
; AVX512F-LABEL: vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,1,8,3,8,5,8,7]
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,1,8,3,8,5,8,7]
+; AVX512F-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -5638,9 +5631,9 @@ define void @vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4(ptr %i
;
; AVX512DQ-LABEL: vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,1,8,3,8,5,8,7]
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,1,8,3,8,5,8,7]
+; AVX512DQ-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -5651,9 +5644,9 @@ define void @vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4(ptr %i
;
; AVX512BW-LABEL: vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,1,8,3,8,5,8,7]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,1,8,3,8,5,8,7]
+; AVX512BW-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -5708,9 +5701,9 @@ define void @vec512_i64_widen_to_i256_factor4_broadcast_to_v2i256_factor2(ptr %i
;
; AVX512F-LABEL: vec512_i64_widen_to_i256_factor4_broadcast_to_v2i256_factor2:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,1,2,3,8,5,6,7]
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,1,2,3,8,5,6,7]
+; AVX512F-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -5721,9 +5714,9 @@ define void @vec512_i64_widen_to_i256_factor4_broadcast_to_v2i256_factor2(ptr %i
;
; AVX512DQ-LABEL: vec512_i64_widen_to_i256_factor4_broadcast_to_v2i256_factor2:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,1,2,3,8,5,6,7]
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,1,2,3,8,5,6,7]
+; AVX512DQ-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -5734,9 +5727,9 @@ define void @vec512_i64_widen_to_i256_factor4_broadcast_to_v2i256_factor2(ptr %i
;
; AVX512BW-LABEL: vec512_i64_widen_to_i256_factor4_broadcast_to_v2i256_factor2:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,1,2,3,8,5,6,7]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,1,2,3,8,5,6,7]
+; AVX512BW-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
@@ -5791,9 +5784,9 @@ define void @vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2(ptr %
;
; AVX512F-LABEL: vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,9,2,3,8,9,6,7]
-; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512F-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,9,2,3,8,9,6,7]
+; AVX512F-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512F-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512F-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -5804,9 +5797,9 @@ define void @vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2(ptr %
;
; AVX512DQ-LABEL: vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2:
; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,9,2,3,8,9,6,7]
-; AVX512DQ-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512DQ-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512DQ-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512DQ-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,9,2,3,8,9,6,7]
+; AVX512DQ-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512DQ-NEXT: vextracti64x4 $1, %zmm1, %ymm0
; AVX512DQ-NEXT: vpaddb 32(%rsi), %ymm0, %ymm0
; AVX512DQ-NEXT: vpaddb (%rsi), %ymm1, %ymm1
@@ -5817,9 +5810,9 @@ define void @vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2(ptr %
;
; AVX512BW-LABEL: vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm0 = [8,9,2,3,8,9,6,7]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpermt2q (%rdi), %zmm0, %zmm1
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovsxbq {{.*#+}} zmm1 = [8,9,2,3,8,9,6,7]
+; AVX512BW-NEXT: vpermi2q (%rdi), %zmm0, %zmm1
; AVX512BW-NEXT: vpaddb (%rsi), %zmm1, %zmm0
; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rdx)
; AVX512BW-NEXT: vzeroupper
More information about the llvm-commits
mailing list