[llvm] [WIP][X86] Replace custom ADD/FADD reduction pattern matching with ISD::VECREDUCE_ADD/FADD support (PR #198893)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 22 04:20:51 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/198893
>From 459f174a5e4fc25f985bb8e3a763dec6d72d8534 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Wed, 20 May 2026 21:30:30 +0100
Subject: [PATCH] [WIP][X86] Replace custom ADD/FADD reduction pattern matching
with ISD::VECREDUCE_ADD/FADD support
Make the most of various horizontal add patterns (HADD/PMADDWD/PSADBW instructions etc.) by matching from ISD::VECREDUCE_ADD/FADD nodes directly instead trying to match to expanded shuffle chains
This also allows us to greatly simplify the X86PartialReduction pass, avoiding the need to match reductions when the middle end can do it for us
Always prefer ISD::VECREDUCE_ADD nodes, and prefer ISD::VECREDUCE_FADD node if the reassoc fmf is enabled.
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 382 +-
llvm/lib/Target/X86/X86PartialReduction.cpp | 94 +-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 3 +
.../X86/avx512-intrinsics-fast-isel.ll | 38 +-
.../CodeGen/X86/buildvec-widen-dotproduct.ll | 6 +-
llvm/test/CodeGen/X86/combine-reductions.ll | 25 +-
llvm/test/CodeGen/X86/dpbusd.ll | 8 +-
llvm/test/CodeGen/X86/dpbusd_const.ll | 4 +-
llvm/test/CodeGen/X86/haddsub.ll | 24 +-
.../test/CodeGen/X86/horizontal-reduce-add.ll | 59 +-
.../CodeGen/X86/horizontal-reduce-fadd.ll | 25 +-
llvm/test/CodeGen/X86/horizontal-sum.ll | 186 +-
llvm/test/CodeGen/X86/insertelement-zero.ll | 41 +-
llvm/test/CodeGen/X86/madd.ll | 130 +-
.../CodeGen/X86/min-legal-vector-width.ll | 12 +-
llvm/test/CodeGen/X86/phaddsub-extract.ll | 193 +-
llvm/test/CodeGen/X86/pr173924.ll | 106 +-
llvm/test/CodeGen/X86/pr62286.ll | 21 +-
llvm/test/CodeGen/X86/sad.ll | 184 +-
llvm/test/CodeGen/X86/vector-compress.ll | 1355 ++--
.../CodeGen/X86/vector-reduce-add-mask.ll | 2009 +++---
.../CodeGen/X86/vector-reduce-add-sext.ll | 477 +-
.../CodeGen/X86/vector-reduce-add-zext.ll | 1420 +++-
llvm/test/CodeGen/X86/vector-reduce-add.ll | 444 +-
llvm/test/CodeGen/X86/vector-reduce-ctpop.ll | 6332 +++++++++--------
.../CodeGen/X86/vector-reduce-fadd-fast.ll | 105 +-
llvm/test/CodeGen/X86/vector-trunc.ll | 233 +-
.../X86/vector-reductions-expanded.ll | 17 +-
28 files changed, 7747 insertions(+), 6186 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 7e9e82ae49f9e..ef279a43dab76 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1096,6 +1096,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_FMUL, MVT::v4f32, Legal);
setOperationAction(ISD::STRICT_FDIV, MVT::v4f32, Legal);
setOperationAction(ISD::STRICT_FSQRT, MVT::v4f32, Legal);
+
+ setOperationAction(ISD::VECREDUCE_FADD, MVT::v4f32, Custom);
}
if (!Subtarget.useSoftFloat() && Subtarget.hasSSE2()) {
@@ -1180,6 +1182,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// SSE41 can use PHMINPOS to perform v16i8/v8i16 minmax reductions.
// Fallback to ReplaceNodeResults for vXi64 reductions on 32-bit targets.
for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::i64}) {
+ setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
@@ -1222,6 +1225,8 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_FSETCCS, MVT::v2f64, Custom);
setOperationAction(ISD::STRICT_FSETCCS, MVT::v4f32, Custom);
+ setOperationAction(ISD::VECREDUCE_FADD, MVT::v2f64, Custom);
+
for (auto VT : { MVT::v16i8, MVT::v8i16, MVT::v4i32 }) {
setOperationAction(ISD::SCALAR_TO_VECTOR, VT, Custom);
setOperationAction(ISD::BUILD_VECTOR, VT, Custom);
@@ -1557,6 +1562,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_FSQRT, MVT::v8f32, Legal);
setOperationAction(ISD::STRICT_FSQRT, MVT::v4f64, Legal);
+ setOperationAction(ISD::VECREDUCE_FADD, MVT::v4f64, Custom);
+ setOperationAction(ISD::VECREDUCE_FADD, MVT::v8f32, Custom);
+
if (!Subtarget.hasAVX512())
setOperationAction(ISD::BITCAST, MVT::v32i1, Custom);
@@ -1571,6 +1579,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
@@ -1951,6 +1960,9 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::STRICT_FSQRT, MVT::v8f64, Legal);
setOperationAction(ISD::STRICT_FP_ROUND, MVT::v8f32, Legal);
+ setOperationAction(ISD::VECREDUCE_FADD, MVT::v8f64, Custom);
+ setOperationAction(ISD::VECREDUCE_FADD, MVT::v16f32, Custom);
+
setTruncStoreAction(MVT::v8i64, MVT::v8i8, Legal);
setTruncStoreAction(MVT::v8i64, MVT::v8i16, Legal);
setTruncStoreAction(MVT::v8i64, MVT::v8i32, Legal);
@@ -2047,6 +2059,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
@@ -2808,6 +2821,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
ISD::ANY_EXTEND_VECTOR_INREG,
ISD::SIGN_EXTEND_VECTOR_INREG,
ISD::ZERO_EXTEND_VECTOR_INREG,
+ ISD::VECREDUCE_ADD,
ISD::SINT_TO_FP,
ISD::UINT_TO_FP,
ISD::FP_TO_SINT,
@@ -29850,7 +29864,7 @@ static SDValue LowerVECREDUCE(SDValue Op, const X86Subtarget &Subtarget,
// Peek through identity elements added by legalisation padding.
APInt HiElts = APInt::getBitsSet(NumSrcElts, NumElts / 2, NumElts);
- if (!DAG.isIdentityElement(BinOp, SDNodeFlags(), Src, HiElts, 1)) {
+ if (!DAG.isIdentityElement(BinOp, Op->getFlags(), Src, HiElts, 1)) {
APInt LoElts = APInt::getLowBitsSet(NumSrcElts, NumElts / 2);
SDValue Lo = peekThroughDemandedElts(Src, LoElts, DAG);
SmallVector<int, 16> Mask(NumSrcElts, -1);
@@ -29863,6 +29877,105 @@ static SDValue LowerVECREDUCE(SDValue Op, const X86Subtarget &Subtarget,
return DAG.getExtractVectorElt(DL, ExtractVT, Src, 0);
}
+static SDValue LowerVECREDUCE_HADD(SDValue Op, const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
+ assert((Op.getOpcode() == ISD::VECREDUCE_ADD ||
+ Op.getOpcode() == ISD::VECREDUCE_FADD) &&
+ "Only (F)ADD reductions supported");
+
+ // Only use (F)HADD opcodes if they aren't microcoded or minimizes codesize.
+ if (!shouldUseHorizontalOp(/*IsSingleSource=*/true, DAG, Subtarget))
+ return SDValue();
+
+ SDValue Src = Op.getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ EVT SrcSVT = SrcVT.getScalarType();
+ EVT ExtractVT = Op.getValueType();
+ unsigned NumSrcElts = SrcVT.getVectorNumElements();
+ SDLoc DL(Op);
+
+ if (SrcVT.getSizeInBits() < 128 || !isPowerOf2_32(NumSrcElts))
+ return SDValue();
+
+ if (!((SrcSVT == MVT::i16 || SrcSVT == MVT::i32) && Subtarget.hasSSSE3()) &&
+ !((SrcSVT == MVT::f32 || SrcSVT == MVT::f64) && Subtarget.hasSSE3()))
+ return SDValue();
+
+ unsigned BinOpcode =
+ Op.getOpcode() == ISD::VECREDUCE_ADD ? ISD::ADD : ISD::FADD;
+ unsigned HorizOpcode =
+ Op.getOpcode() == ISD::VECREDUCE_ADD ? X86ISD::HADD : X86ISD::FHADD;
+
+ while (SrcVT.getSizeInBits() > 128) {
+ SDValue Lo, Hi;
+ std::tie(Lo, Hi) = splitVector(Src, DAG, DL);
+ SrcVT = Lo.getValueType();
+ Src = DAG.getNode(BinOpcode, DL, SrcVT, Lo, Hi);
+ NumSrcElts /= 2;
+ }
+ assert(SrcVT.is128BitVector() && "128-bit reduction expected");
+
+ // Perform horizontal sums until the whole sum is in element 0.
+ unsigned ReductionSteps = Log2_32(NumSrcElts);
+ for (unsigned I = 0; I != ReductionSteps; ++I) {
+ unsigned NumRdxElts = NumSrcElts >> I;
+ APInt HiElts = APInt::getBitsSet(NumSrcElts, NumRdxElts / 2, NumRdxElts);
+ if (DAG.isIdentityElement(BinOpcode, Op->getFlags(), Src, HiElts, 0)) {
+ APInt LoElts = APInt::getLowBitsSet(NumSrcElts, NumRdxElts / 2);
+ Src = peekThroughDemandedElts(Src, LoElts, DAG);
+ } else {
+ Src = DAG.getNode(HorizOpcode, DL, SrcVT, Src, Src);
+ }
+ }
+
+ return DAG.getExtractVectorElt(DL, ExtractVT, Src, 0);
+}
+
+static SDValue LowerVECREDUCE_ADD(SDValue Op, const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
+ SDValue Src = Op.getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ EVT ExtractVT = Op.getValueType();
+ unsigned NumSrcElts = SrcVT.getVectorNumElements();
+ SDLoc DL(Op);
+
+ if (!isPowerOf2_32(NumSrcElts) || NumSrcElts == 1)
+ return LowerVECREDUCE(Op, Subtarget, DAG, /*AllowScalarization=*/true);
+
+ // vXi8 add reduction - sum lo/hi halves then use PSADBW.
+ if (SrcVT.getScalarType() == MVT::i8) {
+ while (SrcVT.getSizeInBits() > 128) {
+ SDValue Lo, Hi;
+ std::tie(Lo, Hi) = splitVector(Src, DAG, DL);
+ SrcVT = Lo.getValueType();
+ Src = DAG.getNode(ISD::ADD, DL, SrcVT, Lo, Hi);
+ }
+ assert(SrcVT == MVT::v16i8 && "v16i8 reduction expected");
+
+ SDValue Hi = DAG.getVectorShuffle(
+ MVT::v16i8, DL, Src, Src,
+ {8, 9, 10, 11, 12, 13, 14, 15, 8, 9, 10, 11, 12, 13, 14, 15});
+ Src = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Src, Hi);
+ Src = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Src,
+ getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
+ Src = DAG.getBitcast(MVT::v16i8, Src);
+ return DAG.getExtractVectorElt(DL, ExtractVT, Src, 0);
+ }
+
+ if (SDValue HADD = LowerVECREDUCE_HADD(Op, Subtarget, DAG))
+ return HADD;
+
+ return LowerVECREDUCE(Op, Subtarget, DAG, /*AllowScalarization=*/true);
+}
+
+static SDValue LowerVECREDUCE_FADD(SDValue Op, const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
+ if (SDValue HADD = LowerVECREDUCE_HADD(Op, Subtarget, DAG))
+ return HADD;
+
+ return LowerVECREDUCE(Op, Subtarget, DAG, /*AllowScalarization=*/true);
+}
+
static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
MVT VT = Op.getSimpleValueType();
@@ -34709,6 +34822,8 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR: return LowerVECREDUCE(Op, Subtarget, DAG, true);
+ case ISD::VECREDUCE_ADD: return LowerVECREDUCE_ADD(Op, Subtarget, DAG);
+ case ISD::VECREDUCE_FADD: return LowerVECREDUCE_FADD(Op, Subtarget, DAG);
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
case ISD::FMINIMUMNUM:
@@ -35511,6 +35626,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
}
return;
}
+ case ISD::VECREDUCE_ADD: {
+ assert(N->getValueType(0) == MVT::i64 && "Unexpected vector reduction");
+ if (SDValue Res = LowerVECREDUCE(SDValue(N, 0), Subtarget, DAG, true))
+ Results.push_back(Res);
+ return;
+ }
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_SMIN:
case ISD::VECREDUCE_UMAX:
@@ -47417,25 +47538,26 @@ static SDValue combinePredicateReduction(SDNode *Extract, SelectionDAG &DAG,
return DAG.getNegative(Zext, DL, ExtractVT);
}
-static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
+static SDValue combineVPDPBUSDPattern(SDNode *N, const SDLoc &DL,
+ SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
if (!Subtarget.hasVNNI() && !Subtarget.hasAVXVNNI())
return SDValue();
- EVT ExtractVT = Extract->getValueType(0);
- // Verify the type we're extracting is i32, as the output element type of
+ if (N->getOpcode() != ISD::VECREDUCE_ADD)
+ return SDValue();
+
+ EVT ExtractVT = N->getValueType(0);
+ // Verify the type we're reducing to is i32, as the output element type of
// vpdpbusd is i32.
if (ExtractVT != MVT::i32)
return SDValue();
- EVT VT = Extract->getOperand(0).getValueType();
+ SDValue Root = N->getOperand(0);
+ EVT VT = Root.getValueType();
if (!isPowerOf2_32(VT.getVectorNumElements()))
return SDValue();
- // Match shuffle + add pyramid.
- ISD::NodeType BinOp;
- SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
-
// We can't combine to vpdpbusd for zext, because each of the 4 multiplies
// done by vpdpbusd compute a signed 16-bit product that will be sign extended
// before adding into the accumulator.
@@ -47447,7 +47569,7 @@ static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
// Root = Root.getOperand(0);
// If there was a match, we want Root to be a mul.
- if (!Root || Root.getOpcode() != ISD::MUL)
+ if (Root.getOpcode() != ISD::MUL)
return SDValue();
// Check whether we have an extend and mul pattern
@@ -47456,7 +47578,6 @@ static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
return SDValue();
// Create the dot product instruction.
- SDLoc DL(Extract);
unsigned StageBias;
SDValue DP = createVPDPBUSD(DAG, LHS, RHS, StageBias, DL, Subtarget);
@@ -47483,12 +47604,11 @@ static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
EVT ResVT =
EVT::getVectorVT(*DAG.getContext(), ExtractVT,
DpVT.getSizeInBits() / ExtractVT.getSizeInBits());
- DP = DAG.getBitcast(ResVT, DP);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, DP,
- Extract->getOperand(1));
+ return DAG.getExtractVectorElt(DL, ExtractVT, DAG.getBitcast(ResVT, DP), 0);
}
-static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
+static SDValue combineBasicSADPattern(SDNode *N, const SDLoc &DL,
+ SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
using namespace SDPatternMatch;
@@ -47496,21 +47616,19 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
if (!Subtarget.hasSSE2())
return SDValue();
- EVT ExtractVT = Extract->getValueType(0);
+ if (N->getOpcode() != ISD::VECREDUCE_ADD)
+ return SDValue();
+
+ EVT ExtractVT = N->getValueType(0);
if (ExtractVT != MVT::i8 && ExtractVT != MVT::i16 && ExtractVT != MVT::i32 &&
ExtractVT != MVT::i64)
return SDValue();
- EVT VT = Extract->getOperand(0).getValueType();
+ SDValue Root = N->getOperand(0);
+ EVT VT = Root.getValueType();
if (!isPowerOf2_32(VT.getVectorNumElements()))
return SDValue();
- // Match shuffle + add pyramid.
- ISD::NodeType BinOp;
- SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
- if (!Root)
- return SDValue();
-
// The operand is expected to be zero extended from i8.
// In order to convert to i64 and above, additional any/zero/sign
// extend is expected.
@@ -47542,7 +47660,6 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
return SDValue();
// Create the SAD instruction.
- SDLoc DL(Extract);
SDValue SAD = createPSADBW(DAG, Src0, Src1, DL, Subtarget);
// If the original vector was wider than 8 elements, sum over the results
@@ -47552,9 +47669,9 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
if (Stages > 3) {
unsigned SadElems = SadVT.getVectorNumElements();
- for(unsigned i = Stages - 3; i > 0; --i) {
+ for (unsigned i = Stages - 3; i > 0; --i) {
SmallVector<int, 16> Mask(SadElems, -1);
- for(unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
+ for (unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
Mask[j] = MaskEnd + j;
SDValue Shuffle =
@@ -47567,9 +47684,7 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
// Return the lowest ExtractSizeInBits bits.
EVT ResVT = EVT::getVectorVT(*DAG.getContext(), ExtractVT,
SadVT.getSizeInBits() / ExtractSizeInBits);
- SAD = DAG.getBitcast(ResVT, SAD);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, SAD,
- Extract->getOperand(1));
+ return DAG.getExtractVectorElt(DL, ExtractVT, DAG.getBitcast(ResVT, SAD), 0);
}
// If this extract is from a loaded vector value and will be used as an
@@ -47934,8 +48049,7 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
return SDValue();
ISD::NodeType Opc;
- SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc,
- {ISD::ADD, ISD::MUL, ISD::FADD}, true);
+ SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc, {ISD::MUL}, true);
if (!Rdx)
return SDValue();
@@ -47950,7 +48064,6 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
SDLoc DL(ExtElt);
unsigned NumElts = VecVT.getVectorNumElements();
- unsigned EltSizeInBits = VecVT.getScalarSizeInBits();
// Extend v4i8/v8i8 vector to v16i8, with undef upper 64-bits.
auto WidenToV16I8 = [&](SDValue V, bool ZeroExtend) {
@@ -47971,146 +48084,36 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
};
// vXi8 mul reduction - promote to vXi16 mul reduction.
- if (Opc == ISD::MUL) {
- if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts))
- return SDValue();
- if (VecVT.getSizeInBits() >= 128) {
- EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);
- SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
- SDValue Hi = getUnpackh(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
- Lo = DAG.getBitcast(WideVT, Lo);
- Hi = DAG.getBitcast(WideVT, Hi);
- Rdx = DAG.getNode(Opc, DL, WideVT, Lo, Hi);
- while (Rdx.getValueSizeInBits() > 128) {
- std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
- Rdx = DAG.getNode(Opc, DL, Lo.getValueType(), Lo, Hi);
- }
- } else {
- Rdx = WidenToV16I8(Rdx, false);
- Rdx = getUnpackl(DAG, DL, MVT::v16i8, Rdx, DAG.getUNDEF(MVT::v16i8));
- Rdx = DAG.getBitcast(MVT::v8i16, Rdx);
- }
- if (NumElts >= 8)
- Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
- DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
- {4, 5, 6, 7, -1, -1, -1, -1}));
- Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
- DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
- {2, 3, -1, -1, -1, -1, -1, -1}));
- Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
- DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
- {1, -1, -1, -1, -1, -1, -1, -1}));
- Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
- }
-
- // vXi8 add reduction - sub 128-bit vector.
- if (VecVT == MVT::v4i8 || VecVT == MVT::v8i8) {
- Rdx = WidenToV16I8(Rdx, true);
- Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
- DAG.getConstant(0, DL, MVT::v16i8));
- Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
- }
-
- // Must be a >=128-bit vector with pow2 elements.
- if ((VecVT.getSizeInBits() % 128) != 0 || !isPowerOf2_32(NumElts))
+ if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts))
return SDValue();
- // vXi8 add reduction - sum lo/hi halves then use PSADBW.
- if (VT == MVT::i8) {
+ if (VecVT.getSizeInBits() >= 128) {
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);
+ SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
+ SDValue Hi = getUnpackh(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
+ Lo = DAG.getBitcast(WideVT, Lo);
+ Hi = DAG.getBitcast(WideVT, Hi);
+ Rdx = DAG.getNode(Opc, DL, WideVT, Lo, Hi);
while (Rdx.getValueSizeInBits() > 128) {
- SDValue Lo, Hi;
std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
- VecVT = Lo.getValueType();
- Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
+ Rdx = DAG.getNode(Opc, DL, Lo.getValueType(), Lo, Hi);
}
- assert(VecVT == MVT::v16i8 && "v16i8 reduction expected");
-
- SDValue Hi = DAG.getVectorShuffle(
- MVT::v16i8, DL, Rdx, Rdx,
- {8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1});
- Rdx = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Rdx, Hi);
- Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
- getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
- Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
- }
-
- // See if we can use vXi8 PSADBW add reduction for larger zext types.
- // If the source vector values are 0-255, then we can use PSADBW to
- // sum+zext v8i8 subvectors to vXi64, then perform the reduction.
- // TODO: See if its worth avoiding vXi16/i32 truncations?
- if (Opc == ISD::ADD && NumElts >= 4 && EltSizeInBits >= 16 &&
- EltSizeInBits <= 64 && DAG.computeKnownBits(Rdx).getMaxValue().ule(255) &&
- (EltSizeInBits == 16 || Rdx.getOpcode() == ISD::ZERO_EXTEND ||
- Subtarget.hasAVX512())) {
- if (Rdx.getValueType() == MVT::v8i16) {
- Rdx = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Rdx,
- DAG.getUNDEF(MVT::v8i16));
- } else {
- EVT ByteVT = VecVT.changeVectorElementType(*DAG.getContext(), MVT::i8);
- Rdx = DAG.getNode(ISD::TRUNCATE, DL, ByteVT, Rdx);
- if (ByteVT.getSizeInBits() < 128)
- Rdx = WidenToV16I8(Rdx, true);
- }
-
- // Build the PSADBW, split as 128/256/512 bits for SSE/AVX2/AVX512BW.
- auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
- ArrayRef<SDValue> Ops) {
- MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);
- SDValue Zero = DAG.getConstant(0, DL, Ops[0].getValueType());
- return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops[0], Zero);
- };
- MVT SadVT = MVT::getVectorVT(MVT::i64, Rdx.getValueSizeInBits() / 64);
- Rdx = SplitOpsAndApply(DAG, Subtarget, DL, SadVT, {Rdx}, PSADBWBuilder);
-
- // TODO: We could truncate to vXi16/vXi32 before performing the reduction.
- while (Rdx.getValueSizeInBits() > 128) {
- SDValue Lo, Hi;
- std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
- VecVT = Lo.getValueType();
- Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
- }
- assert(Rdx.getValueType() == MVT::v2i64 && "v2i64 reduction expected");
-
- if (NumElts > 8) {
- SDValue RdxHi = DAG.getVectorShuffle(MVT::v2i64, DL, Rdx, Rdx, {1, -1});
- Rdx = DAG.getNode(ISD::ADD, DL, MVT::v2i64, Rdx, RdxHi);
- }
-
- VecVT = MVT::getVectorVT(VT.getSimpleVT(), 128 / VT.getSizeInBits());
- Rdx = DAG.getBitcast(VecVT, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
- }
-
- // Only use (F)HADD opcodes if they aren't microcoded or minimizes codesize.
- if (!shouldUseHorizontalOp(true, DAG, Subtarget))
- return SDValue();
-
- unsigned HorizOpcode = Opc == ISD::ADD ? X86ISD::HADD : X86ISD::FHADD;
-
- // 256-bit horizontal instructions operate on 128-bit chunks rather than
- // across the whole vector, so we need an extract + hop preliminary stage.
- // This is the only step where the operands of the hop are not the same value.
- // TODO: We could extend this to handle 512-bit or even longer vectors.
- if (((VecVT == MVT::v16i16 || VecVT == MVT::v8i32) && Subtarget.hasSSSE3()) ||
- ((VecVT == MVT::v8f32 || VecVT == MVT::v4f64) && Subtarget.hasSSE3())) {
- unsigned NumElts = VecVT.getVectorNumElements();
- SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL);
- SDValue Lo = extract128BitVector(Rdx, 0, DAG, DL);
- Rdx = DAG.getNode(HorizOpcode, DL, Lo.getValueType(), Hi, Lo);
- VecVT = Rdx.getValueType();
+ } else {
+ Rdx = WidenToV16I8(Rdx, false);
+ Rdx = getUnpackl(DAG, DL, MVT::v16i8, Rdx, DAG.getUNDEF(MVT::v16i8));
+ Rdx = DAG.getBitcast(MVT::v8i16, Rdx);
}
- if (!((VecVT == MVT::v8i16 || VecVT == MVT::v4i32) && Subtarget.hasSSSE3()) &&
- !((VecVT == MVT::v4f32 || VecVT == MVT::v2f64) && Subtarget.hasSSE3()))
- return SDValue();
-
- // extract (add (shuf X), X), 0 --> extract (hadd X, X), 0
- unsigned ReductionSteps = Log2_32(VecVT.getVectorNumElements());
- for (unsigned i = 0; i != ReductionSteps; ++i)
- Rdx = DAG.getNode(HorizOpcode, DL, VecVT, Rdx, Rdx);
-
+ if (NumElts >= 8)
+ Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
+ DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
+ {4, 5, 6, 7, -1, -1, -1, -1}));
+ Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
+ DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
+ {2, 3, -1, -1, -1, -1, -1, -1}));
+ Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
+ DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
+ {1, -1, -1, -1, -1, -1, -1, -1}));
+ Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
}
@@ -48216,20 +48219,11 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32,
InputVector.getOperand(0));
- // Check whether this extract is the root of a sum of absolute differences
- // pattern. This has to be done here because we really want it to happen
- // pre-legalization,
- if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget))
- return SAD;
-
- if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DAG, Subtarget))
- return VPDPBUSD;
-
// Attempt to replace an all_of/any_of horizontal reduction with a MOVMSK.
if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget))
return Cmp;
- // Attempt to optimize ADD/FADD/MUL reductions with HADD, promotion etc..
+ // Attempt to optimize vXi8 MUL reductions.
if (SDValue V = combineArithReduction(N, DAG, Subtarget))
return V;
@@ -48318,6 +48312,41 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+static SDValue combineVECREDUCE_ADD(SDNode *N, SelectionDAG &DAG,
+ const X86Subtarget &Subtarget) {
+ SDLoc DL(N);
+
+ if (SDValue SAD = combineBasicSADPattern(N, DL, DAG, Subtarget))
+ return SAD;
+
+ if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DL, DAG, Subtarget))
+ return VPDPBUSD;
+
+ SDValue Src = N->getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ EVT SrcSVT = SrcVT.getScalarType();
+ unsigned EltSizeInBits = SrcSVT.getSizeInBits();
+ unsigned NumSrcElts = SrcVT.getVectorNumElements();
+
+ if (NumSrcElts >= 4 && NumSrcElts <= 16 && EltSizeInBits > 8) {
+ KnownBits KnownSrc = DAG.computeKnownBits(Src);
+ if (KnownSrc.getMaxValue().ule(255)) {
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(), SrcSVT, 16);
+ Src = DAG.getInsertSubvector(DL, DAG.getConstant(0, DL, WideVT), Src, 0);
+ Src = DAG.getNode(ISD::TRUNCATE, DL, MVT::v16i8, Src);
+ Src = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Src,
+ DAG.getConstant(0, DL, MVT::v16i8));
+ if (NumSrcElts <= 8)
+ Src = DAG.getExtractVectorElt(DL, MVT::i64, Src, 0);
+ else
+ Src = DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i64, Src);
+ return DAG.getZExtOrTrunc(Src, DL, N->getValueType(0));
+ }
+ }
+
+ return SDValue();
+}
+
// Convert (vXiY *ext(vXi1 bitcast(iX))) to extend_in_reg(broadcast(iX)).
// This is more or less the reverse of combineBitcastvxi1.
static SDValue combineToExtendBoolVectorInReg(
@@ -63236,6 +63265,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case X86ISD::VFMULC: return combineFMulcFCMulc(N, DAG, Subtarget);
case ISD::FNEG: return combineFneg(N, DAG, DCI, Subtarget);
case ISD::TRUNCATE: return combineTruncate(N, DAG, Subtarget);
+ case ISD::VECREDUCE_ADD: return combineVECREDUCE_ADD(N, DAG, Subtarget);
case X86ISD::VTRUNC: return combineVTRUNC(N, DAG, DCI);
case X86ISD::VTRUNCS:
case X86ISD::VTRUNCUS: return combineVTRUNCSAT(N, DAG, DCI, Subtarget);
diff --git a/llvm/lib/Target/X86/X86PartialReduction.cpp b/llvm/lib/Target/X86/X86PartialReduction.cpp
index a017ed9f2ff50..412825fa45b1c 100644
--- a/llvm/lib/Target/X86/X86PartialReduction.cpp
+++ b/llvm/lib/Target/X86/X86PartialReduction.cpp
@@ -42,7 +42,7 @@ class X86PartialReduction {
bool run(Function &F);
private:
- bool tryMAddReplacement(Instruction *Op, bool ReduceInOneBB);
+ bool tryMAddReplacement(Instruction *Op);
bool trySADReplacement(Instruction *Op);
};
@@ -106,8 +106,7 @@ static bool matchVPDPBUSDPattern(const X86Subtarget *ST, BinaryOperator *Mul,
return false;
}
-bool X86PartialReduction::tryMAddReplacement(Instruction *Op,
- bool ReduceInOneBB) {
+bool X86PartialReduction::tryMAddReplacement(Instruction *Op) {
if (!ST->hasSSE2())
return false;
@@ -128,9 +127,7 @@ bool X86PartialReduction::tryMAddReplacement(Instruction *Op,
// If the target support VNNI, leave it to ISel to combine reduce operation
// to VNNI instruction.
- // TODO: we can support transforming reduce to VNNI intrinsic for across block
- // in this pass.
- if (ReduceInOneBB && matchVPDPBUSDPattern(ST, Mul, DL))
+ if (matchVPDPBUSDPattern(ST, Mul, DL))
return false;
// LHS and RHS should be only used once or if they are the same then only
@@ -353,67 +350,6 @@ bool X86PartialReduction::trySADReplacement(Instruction *Op) {
return true;
}
-// Walk backwards from the ExtractElementInst and determine if it is the end of
-// a horizontal reduction. Return the input to the reduction if we find one.
-static Value *matchAddReduction(const ExtractElementInst &EE,
- bool &ReduceInOneBB) {
- ReduceInOneBB = true;
- // Make sure we're extracting index 0.
- auto *Index = dyn_cast<ConstantInt>(EE.getIndexOperand());
- if (!Index || !Index->isNullValue())
- return nullptr;
-
- const auto *BO = dyn_cast<BinaryOperator>(EE.getVectorOperand());
- if (!BO || BO->getOpcode() != Instruction::Add || !BO->hasOneUse())
- return nullptr;
- if (EE.getParent() != BO->getParent())
- ReduceInOneBB = false;
-
- unsigned NumElems = cast<FixedVectorType>(BO->getType())->getNumElements();
- // Ensure the reduction size is a power of 2.
- if (!isPowerOf2_32(NumElems))
- return nullptr;
-
- const Value *Op = BO;
- unsigned Stages = Log2_32(NumElems);
- for (unsigned i = 0; i != Stages; ++i) {
- const auto *BO = dyn_cast<BinaryOperator>(Op);
- if (!BO || BO->getOpcode() != Instruction::Add)
- return nullptr;
- if (EE.getParent() != BO->getParent())
- ReduceInOneBB = false;
-
- // If this isn't the first add, then it should only have 2 users, the
- // shuffle and another add which we checked in the previous iteration.
- if (i != 0 && !BO->hasNUses(2))
- return nullptr;
-
- Value *LHS = BO->getOperand(0);
- Value *RHS = BO->getOperand(1);
-
- auto *Shuffle = dyn_cast<ShuffleVectorInst>(LHS);
- if (Shuffle) {
- Op = RHS;
- } else {
- Shuffle = dyn_cast<ShuffleVectorInst>(RHS);
- Op = LHS;
- }
-
- // The first operand of the shuffle should be the same as the other operand
- // of the bin op.
- if (!Shuffle || Shuffle->getOperand(0) != Op)
- return nullptr;
-
- // Verify the shuffle has the expected (at this stage of the pyramid) mask.
- unsigned MaskEnd = 1 << i;
- for (unsigned Index = 0; Index < MaskEnd; ++Index)
- if (Shuffle->getMaskValue(Index) != (int)(MaskEnd + Index))
- return nullptr;
- }
-
- return const_cast<Value *>(Op);
-}
-
// See if this BO is reachable from this Phi by walking forward through single
// use BinaryOperators with the same opcode. If we get back then we know we've
// found a loop and it is safe to step through this Add to find more leaves.
@@ -447,9 +383,7 @@ static void collectLeaves(Value *Root, SmallVectorImpl<Instruction *> &Leaves) {
continue;
if (auto *PN = dyn_cast<PHINode>(V)) {
- // PHI node should have single use unless it is the root node, then it
- // has 2 uses.
- if (!PN->hasNUses(PN == Root ? 2 : 1))
+ if (!PN->hasNUses(1))
break;
// Push incoming values to the worklist.
@@ -461,14 +395,14 @@ static void collectLeaves(Value *Root, SmallVectorImpl<Instruction *> &Leaves) {
if (auto *BO = dyn_cast<BinaryOperator>(V)) {
if (BO->getOpcode() == Instruction::Add) {
// Simple case. Single use, just push its operands to the worklist.
- if (BO->hasNUses(BO == Root ? 2 : 1)) {
+ if (BO->hasNUses(1)) {
append_range(Worklist, BO->operands());
continue;
}
// If there is additional use, make sure it is an unvisited phi that
// gets us back to this node.
- if (BO->hasNUses(BO == Root ? 3 : 2)) {
+ if (BO->hasNUses(2)) {
PHINode *PN = nullptr;
for (auto *U : BO->users())
if (auto *P = dyn_cast<PHINode>(U))
@@ -492,7 +426,7 @@ static void collectLeaves(Value *Root, SmallVectorImpl<Instruction *> &Leaves) {
// Not an add or phi, make it a leaf.
if (auto *I = dyn_cast<Instruction>(V)) {
- if (!V->hasNUses(I == Root ? 2 : 1))
+ if (!V->hasNUses(1))
continue;
// Add this as a leaf.
@@ -508,22 +442,18 @@ bool X86PartialReduction::run(Function &F) {
bool MadeChange = false;
for (auto &BB : F) {
for (auto &I : BB) {
- auto *EE = dyn_cast<ExtractElementInst>(&I);
- if (!EE)
- continue;
-
- bool ReduceInOneBB;
// First find a reduction tree.
- // FIXME: Do we need to handle other opcodes than Add?
- Value *Root = matchAddReduction(*EE, ReduceInOneBB);
- if (!Root)
+ // FIXME: Do we need to handle other reduction opcodes than Add?
+ auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II || II->getIntrinsicID() != Intrinsic::vector_reduce_add)
continue;
+ Value *Root = II->getOperand(0);
SmallVector<Instruction *, 8> Leaves;
collectLeaves(Root, Leaves);
for (Instruction *I : Leaves) {
- if (tryMAddReplacement(I, ReduceInOneBB)) {
+ if (tryMAddReplacement(I)) {
MadeChange = true;
continue;
}
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 0e9d8679ca9ec..fd93bb00b97db 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -6828,11 +6828,14 @@ bool X86TTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
switch (II->getIntrinsicID()) {
default:
return true;
+ case Intrinsic::vector_reduce_add:
case Intrinsic::vector_reduce_smax:
case Intrinsic::vector_reduce_smin:
case Intrinsic::vector_reduce_umax:
case Intrinsic::vector_reduce_umin:
return false;
+ case Intrinsic::vector_reduce_fadd:
+ return !II->getFastMathFlags().allowReassoc();
}
}
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
index 1e14256db3584..64957514707ea 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
@@ -6519,7 +6519,7 @@ define i64 @test_mm512_reduce_add_epi64(<8 x i64> %__W) nounwind {
; X86-LABEL: test_mm512_reduce_add_epi64:
; X86: # %bb.0: # %entry
; X86-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; X86-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; X86-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; X86-NEXT: vextracti128 $1, %ymm0, %xmm1
; X86-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -6532,7 +6532,7 @@ define i64 @test_mm512_reduce_add_epi64(<8 x i64> %__W) nounwind {
; X64-LABEL: test_mm512_reduce_add_epi64:
; X64: # %bb.0: # %entry
; X64-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; X64-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; X64-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; X64-NEXT: vextracti128 $1, %ymm0, %xmm1
; X64-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -6684,7 +6684,7 @@ define i64 @test_mm512_mask_reduce_add_epi64(i8 zeroext %__M, <8 x i64> %__W) no
; X86-NEXT: kmovw %eax, %k1
; X86-NEXT: vmovdqa64 %zmm0, %zmm0 {%k1} {z}
; X86-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; X86-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; X86-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; X86-NEXT: vextracti128 $1, %ymm0, %xmm1
; X86-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -6699,7 +6699,7 @@ define i64 @test_mm512_mask_reduce_add_epi64(i8 zeroext %__M, <8 x i64> %__W) no
; X64-NEXT: kmovw %edi, %k1
; X64-NEXT: vmovdqa64 %zmm0, %zmm0 {%k1} {z}
; X64-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; X64-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; X64-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; X64-NEXT: vextracti128 $1, %ymm0, %xmm1
; X64-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -6875,7 +6875,7 @@ define i32 @test_mm512_reduce_add_epi32(<8 x i64> %__W) nounwind {
; CHECK-LABEL: test_mm512_reduce_add_epi32:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -6958,7 +6958,7 @@ define i32 @test_mm512_mask_reduce_add_epi32(i16 zeroext %__M, <8 x i64> %__W) n
; X86-NEXT: kmovw %eax, %k1
; X86-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
; X86-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; X86-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; X86-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; X86-NEXT: vextracti128 $1, %ymm0, %xmm1
; X86-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; X86-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -6974,7 +6974,7 @@ define i32 @test_mm512_mask_reduce_add_epi32(i16 zeroext %__M, <8 x i64> %__W) n
; X64-NEXT: kmovw %edi, %k1
; X64-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
; X64-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; X64-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; X64-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; X64-NEXT: vextracti128 $1, %ymm0, %xmm1
; X64-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; X64-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -7127,7 +7127,7 @@ define double @test_mm512_reduce_add_pd(<8 x double> %__W) nounwind {
; X86-NEXT: andl $-8, %esp
; X86-NEXT: subl $8, %esp
; X86-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; X86-NEXT: vaddpd %zmm1, %zmm0, %zmm0
+; X86-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; X86-NEXT: vextractf128 $1, %ymm0, %xmm1
; X86-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; X86-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
@@ -7142,13 +7142,11 @@ define double @test_mm512_reduce_add_pd(<8 x double> %__W) nounwind {
; X64-LABEL: test_mm512_reduce_add_pd:
; X64: # %bb.0: # %entry
; X64-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; X64-NEXT: vaddpd %zmm1, %zmm0, %zmm0
+; X64-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; X64-NEXT: vextractf128 $1, %ymm0, %xmm1
; X64-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; X64-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
; X64-NEXT: vaddsd %xmm1, %xmm0, %xmm0
-; X64-NEXT: vmovsd {{.*#+}} xmm1 = [-0.0E+0,0.0E+0]
-; X64-NEXT: vaddsd %xmm0, %xmm1, %xmm0
; X64-NEXT: vzeroupper
; X64-NEXT: retq
entry:
@@ -7198,7 +7196,7 @@ define float @test_mm512_reduce_add_ps(<16 x float> %__W) nounwind {
; X86: # %bb.0: # %entry
; X86-NEXT: pushl %eax
; X86-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; X86-NEXT: vaddps %zmm1, %zmm0, %zmm0
+; X86-NEXT: vaddps %ymm1, %ymm0, %ymm0
; X86-NEXT: vextractf128 $1, %ymm0, %xmm1
; X86-NEXT: vaddps %xmm1, %xmm0, %xmm0
; X86-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
@@ -7214,15 +7212,13 @@ define float @test_mm512_reduce_add_ps(<16 x float> %__W) nounwind {
; X64-LABEL: test_mm512_reduce_add_ps:
; X64: # %bb.0: # %entry
; X64-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; X64-NEXT: vaddps %zmm1, %zmm0, %zmm0
+; X64-NEXT: vaddps %ymm1, %ymm0, %ymm0
; X64-NEXT: vextractf128 $1, %ymm0, %xmm1
; X64-NEXT: vaddps %xmm1, %xmm0, %xmm0
; X64-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
; X64-NEXT: vaddps %xmm1, %xmm0, %xmm0
; X64-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; X64-NEXT: vaddss %xmm1, %xmm0, %xmm0
-; X64-NEXT: vmovss {{.*#+}} xmm1 = [-0.0E+0,0.0E+0,0.0E+0,0.0E+0]
-; X64-NEXT: vaddss %xmm0, %xmm1, %xmm0
; X64-NEXT: vzeroupper
; X64-NEXT: retq
entry:
@@ -7278,7 +7274,7 @@ define double @test_mm512_mask_reduce_add_pd(i8 zeroext %__M, <8 x double> %__W)
; X86-NEXT: kmovw %eax, %k1
; X86-NEXT: vmovapd %zmm0, %zmm0 {%k1} {z}
; X86-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; X86-NEXT: vaddpd %zmm1, %zmm0, %zmm0
+; X86-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; X86-NEXT: vextractf128 $1, %ymm0, %xmm1
; X86-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; X86-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
@@ -7295,13 +7291,11 @@ define double @test_mm512_mask_reduce_add_pd(i8 zeroext %__M, <8 x double> %__W)
; X64-NEXT: kmovw %edi, %k1
; X64-NEXT: vmovapd %zmm0, %zmm0 {%k1} {z}
; X64-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; X64-NEXT: vaddpd %zmm1, %zmm0, %zmm0
+; X64-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; X64-NEXT: vextractf128 $1, %ymm0, %xmm1
; X64-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; X64-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
; X64-NEXT: vaddsd %xmm1, %xmm0, %xmm0
-; X64-NEXT: vmovsd {{.*#+}} xmm1 = [-0.0E+0,0.0E+0]
-; X64-NEXT: vaddsd %xmm0, %xmm1, %xmm0
; X64-NEXT: vzeroupper
; X64-NEXT: retq
entry:
@@ -7365,7 +7359,7 @@ define float @test_mm512_mask_reduce_add_ps(i16 zeroext %__M, <16 x float> %__W)
; X86-NEXT: kmovw %eax, %k1
; X86-NEXT: vmovaps %zmm0, %zmm0 {%k1} {z}
; X86-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; X86-NEXT: vaddps %zmm1, %zmm0, %zmm0
+; X86-NEXT: vaddps %ymm1, %ymm0, %ymm0
; X86-NEXT: vextractf128 $1, %ymm0, %xmm1
; X86-NEXT: vaddps %xmm1, %xmm0, %xmm0
; X86-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
@@ -7383,15 +7377,13 @@ define float @test_mm512_mask_reduce_add_ps(i16 zeroext %__M, <16 x float> %__W)
; X64-NEXT: kmovw %edi, %k1
; X64-NEXT: vmovaps %zmm0, %zmm0 {%k1} {z}
; X64-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; X64-NEXT: vaddps %zmm1, %zmm0, %zmm0
+; X64-NEXT: vaddps %ymm1, %ymm0, %ymm0
; X64-NEXT: vextractf128 $1, %ymm0, %xmm1
; X64-NEXT: vaddps %xmm1, %xmm0, %xmm0
; X64-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
; X64-NEXT: vaddps %xmm1, %xmm0, %xmm0
; X64-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; X64-NEXT: vaddss %xmm1, %xmm0, %xmm0
-; X64-NEXT: vmovss {{.*#+}} xmm1 = [-0.0E+0,0.0E+0,0.0E+0,0.0E+0]
-; X64-NEXT: vaddss %xmm0, %xmm1, %xmm0
; X64-NEXT: vzeroupper
; X64-NEXT: retq
entry:
diff --git a/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll b/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll
index 07c81c6f8b7a4..336b971c23200 100644
--- a/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll
+++ b/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll
@@ -417,8 +417,7 @@ define i32 @dot_ext_v2i16_v2i32(ptr %a, i64 %a_stride, ptr %b) nounwind {
; SSE4: # %bb.0:
; SSE4-NEXT: movzwl (%rdi), %eax
; SSE4-NEXT: movd %eax, %xmm0
-; SSE4-NEXT: pinsrw $1, (%rdi,%rsi), %xmm0
-; SSE4-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE4-NEXT: pinsrw $2, (%rdi,%rsi), %xmm0
; SSE4-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
; SSE4-NEXT: pmovsxwd %xmm1, %xmm1
; SSE4-NEXT: pmulld %xmm0, %xmm1
@@ -431,8 +430,7 @@ define i32 @dot_ext_v2i16_v2i32(ptr %a, i64 %a_stride, ptr %b) nounwind {
; AVX: # %bb.0:
; AVX-NEXT: movzwl (%rdi), %eax
; AVX-NEXT: vmovd %eax, %xmm0
-; AVX-NEXT: vpinsrw $1, (%rdi,%rsi), %xmm0, %xmm0
-; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT: vpinsrw $2, (%rdi,%rsi), %xmm0, %xmm0
; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
; AVX-NEXT: vpmovsxwd %xmm1, %xmm1
; AVX-NEXT: vpmulld %xmm0, %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/combine-reductions.ll b/llvm/test/CodeGen/X86/combine-reductions.ll
index 844cd13e71d49..51ca5c34c9882 100644
--- a/llvm/test/CodeGen/X86/combine-reductions.ll
+++ b/llvm/test/CodeGen/X86/combine-reductions.ll
@@ -91,17 +91,10 @@ define { i16, i16 } @test_reduce_v16i16_with_add(<16 x i16> %x, <16 x i16> %y) {
; SSE41-FAST: # %bb.0:
; SSE41-FAST-NEXT: movdqa %xmm0, %xmm4
; SSE41-FAST-NEXT: paddw %xmm1, %xmm4
-; SSE41-FAST-NEXT: pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; SSE41-FAST-NEXT: paddw %xmm4, %xmm5
-; SSE41-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,1,1]
-; SSE41-FAST-NEXT: paddw %xmm5, %xmm4
; SSE41-FAST-NEXT: phaddw %xmm4, %xmm4
-; SSE41-FAST-NEXT: movdqa %xmm1, %xmm5
-; SSE41-FAST-NEXT: phaddw %xmm0, %xmm5
-; SSE41-FAST-NEXT: phaddw %xmm5, %xmm5
-; SSE41-FAST-NEXT: phaddw %xmm5, %xmm5
-; SSE41-FAST-NEXT: phaddw %xmm5, %xmm5
-; SSE41-FAST-NEXT: movd %xmm5, %eax
+; SSE41-FAST-NEXT: phaddw %xmm4, %xmm4
+; SSE41-FAST-NEXT: phaddw %xmm4, %xmm4
+; SSE41-FAST-NEXT: movd %xmm4, %eax
; SSE41-FAST-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
; SSE41-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
; SSE41-FAST-NEXT: pcmpeqw %xmm4, %xmm1
@@ -129,6 +122,7 @@ define { i16, i16 } @test_reduce_v16i16_with_add(<16 x i16> %x, <16 x i16> %y) {
; AVX2-SLOW-NEXT: vpsrld $16, %xmm2, %xmm3
; AVX2-SLOW-NEXT: vpaddw %xmm3, %xmm2, %xmm2
; AVX2-SLOW-NEXT: vmovd %xmm2, %eax
+; AVX2-SLOW-NEXT: vmovd %eax, %xmm2
; AVX2-SLOW-NEXT: vpbroadcastw %xmm2, %ymm2
; AVX2-SLOW-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
; AVX2-SLOW-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
@@ -146,18 +140,13 @@ define { i16, i16 } @test_reduce_v16i16_with_add(<16 x i16> %x, <16 x i16> %y) {
; AVX2-FAST-LABEL: test_reduce_v16i16_with_add:
; AVX2-FAST: # %bb.0:
; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX2-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm3
-; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; AVX2-FAST-NEXT: vpaddw %xmm4, %xmm3, %xmm3
-; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
-; AVX2-FAST-NEXT: vpaddw %xmm4, %xmm3, %xmm3
-; AVX2-FAST-NEXT: vphaddw %xmm3, %xmm3, %xmm3
-; AVX2-FAST-NEXT: vphaddw %xmm0, %xmm2, %xmm2
+; AVX2-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm2
; AVX2-FAST-NEXT: vphaddw %xmm2, %xmm2, %xmm2
; AVX2-FAST-NEXT: vphaddw %xmm2, %xmm2, %xmm2
; AVX2-FAST-NEXT: vphaddw %xmm2, %xmm2, %xmm2
; AVX2-FAST-NEXT: vmovd %xmm2, %eax
-; AVX2-FAST-NEXT: vpbroadcastw %xmm3, %ymm2
+; AVX2-FAST-NEXT: vmovd %eax, %xmm2
+; AVX2-FAST-NEXT: vpbroadcastw %xmm2, %ymm2
; AVX2-FAST-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
; AVX2-FAST-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; AVX2-FAST-NEXT: vpxor %ymm2, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/dpbusd.ll b/llvm/test/CodeGen/X86/dpbusd.ll
index 7bd22d57347b3..d1781fb31ba64 100644
--- a/llvm/test/CodeGen/X86/dpbusd.ll
+++ b/llvm/test/CodeGen/X86/dpbusd.ll
@@ -111,7 +111,7 @@ define i32 @mul_zext(ptr%a, ptr%b, i32 %c, i32 %n) {
; AVXVNNI-AVX512-NEXT: vpmullw %ymm0, %ymm1, %ymm0
; AVXVNNI-AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVXVNNI-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVXVNNI-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVXVNNI-AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVXVNNI-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVXVNNI-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -130,7 +130,7 @@ define i32 @mul_zext(ptr%a, ptr%b, i32 %c, i32 %n) {
; AVX512-NEXT: vpmullw %ymm0, %ymm1, %ymm0
; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -184,7 +184,7 @@ define i32 @mul_sext(ptr%a, ptr%b, i32 %c, i32 %n) {
; AVXVNNI-AVX512-NEXT: vpmullw %ymm0, %ymm1, %ymm0
; AVXVNNI-AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
; AVXVNNI-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVXVNNI-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVXVNNI-AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVXVNNI-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVXVNNI-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -203,7 +203,7 @@ define i32 @mul_sext(ptr%a, ptr%b, i32 %c, i32 %n) {
; AVX512-NEXT: vpmullw %ymm0, %ymm1, %ymm0
; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/dpbusd_const.ll b/llvm/test/CodeGen/X86/dpbusd_const.ll
index 1d6c3f7c5c6e8..bb47df59eefad 100644
--- a/llvm/test/CodeGen/X86/dpbusd_const.ll
+++ b/llvm/test/CodeGen/X86/dpbusd_const.ll
@@ -8,8 +8,8 @@ define i32 @mul_4xi8_zc_exceed(<4 x i8> %a, i32 %c) {
; CHECK-LABEL: mul_4xi8_zc_exceed:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; CHECK-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [0,0,1,0,2,0,128,0]
; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; CHECK-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [2,128,2,128]
; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0
@@ -180,8 +180,8 @@ define i32 @mul_4xi8_cs_exceed(<4 x i8> %a, i32 %c) {
; CHECK-LABEL: mul_4xi8_cs_exceed:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vpmovsxbd %xmm0, %xmm0
+; CHECK-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [0,0,1,0,2,0,256,0]
; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; CHECK-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [2,256,2,256]
; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/haddsub.ll b/llvm/test/CodeGen/X86/haddsub.ll
index a0778195b5c73..b7e82371448be 100644
--- a/llvm/test/CodeGen/X86/haddsub.ll
+++ b/llvm/test/CodeGen/X86/haddsub.ll
@@ -1638,10 +1638,10 @@ define float @fadd_reduce_v8f32(float %a0, <8 x float> %a1) {
;
; SSE3-FAST-LABEL: fadd_reduce_v8f32:
; SSE3-FAST: # %bb.0:
-; SSE3-FAST-NEXT: haddps %xmm1, %xmm2
-; SSE3-FAST-NEXT: haddps %xmm2, %xmm2
-; SSE3-FAST-NEXT: haddps %xmm2, %xmm2
-; SSE3-FAST-NEXT: addss %xmm2, %xmm0
+; SSE3-FAST-NEXT: addps %xmm2, %xmm1
+; SSE3-FAST-NEXT: haddps %xmm1, %xmm1
+; SSE3-FAST-NEXT: haddps %xmm1, %xmm1
+; SSE3-FAST-NEXT: addss %xmm1, %xmm0
; SSE3-FAST-NEXT: retq
;
; AVX-SLOW-LABEL: fadd_reduce_v8f32:
@@ -1659,7 +1659,7 @@ define float @fadd_reduce_v8f32(float %a0, <8 x float> %a1) {
; AVX-FAST-LABEL: fadd_reduce_v8f32:
; AVX-FAST: # %bb.0:
; AVX-FAST-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX-FAST-NEXT: vhaddps %xmm1, %xmm2, %xmm1
+; AVX-FAST-NEXT: vaddps %xmm2, %xmm1, %xmm1
; AVX-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
; AVX-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
; AVX-FAST-NEXT: vaddss %xmm1, %xmm0, %xmm0
@@ -1681,9 +1681,9 @@ define double @fadd_reduce_v4f64(double %a0, <4 x double> %a1) {
;
; SSE3-FAST-LABEL: fadd_reduce_v4f64:
; SSE3-FAST: # %bb.0:
-; SSE3-FAST-NEXT: haddpd %xmm1, %xmm2
-; SSE3-FAST-NEXT: haddpd %xmm2, %xmm2
-; SSE3-FAST-NEXT: addsd %xmm2, %xmm0
+; SSE3-FAST-NEXT: addpd %xmm2, %xmm1
+; SSE3-FAST-NEXT: haddpd %xmm1, %xmm1
+; SSE3-FAST-NEXT: addsd %xmm1, %xmm0
; SSE3-FAST-NEXT: retq
;
; AVX-SLOW-LABEL: fadd_reduce_v4f64:
@@ -1699,7 +1699,7 @@ define double @fadd_reduce_v4f64(double %a0, <4 x double> %a1) {
; AVX-FAST-LABEL: fadd_reduce_v4f64:
; AVX-FAST: # %bb.0:
; AVX-FAST-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX-FAST-NEXT: vhaddpd %xmm1, %xmm2, %xmm1
+; AVX-FAST-NEXT: vaddpd %xmm2, %xmm1, %xmm1
; AVX-FAST-NEXT: vhaddpd %xmm1, %xmm1, %xmm1
; AVX-FAST-NEXT: vaddsd %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT: vzeroupper
@@ -2067,7 +2067,8 @@ define float @partial_reduction_fadd_v8f32(<8 x float> %x) {
;
; AVX-FAST-LABEL: partial_reduction_fadd_v8f32:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX-FAST-NEXT: vzeroupper
; AVX-FAST-NEXT: retq
@@ -2153,7 +2154,8 @@ define float @partial_reduction_fadd_v16f32(<16 x float> %x) {
;
; AVX-FAST-LABEL: partial_reduction_fadd_v16f32:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX-FAST-NEXT: vzeroupper
; AVX-FAST-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-add.ll b/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
index 873083a0703da..1e9c64ef98a7e 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-add.ll
@@ -29,9 +29,10 @@ define i32 @PR37890_v4i32(<4 x i32> %a) {
;
; SSSE3-FAST-LABEL: PR37890_v4i32:
; SSSE3-FAST: # %bb.0:
-; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm0
-; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm0
-; SSSE3-FAST-NEXT: movd %xmm0, %eax
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSSE3-FAST-NEXT: paddd %xmm0, %xmm1
+; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm1
+; SSSE3-FAST-NEXT: movd %xmm1, %eax
; SSSE3-FAST-NEXT: retq
;
; AVX1-SLOW-LABEL: PR37890_v4i32:
@@ -45,7 +46,8 @@ define i32 @PR37890_v4i32(<4 x i32> %a) {
;
; AVX1-FAST-LABEL: PR37890_v4i32:
; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: retq
@@ -96,8 +98,10 @@ define i16 @PR37890_v8i16(<8 x i16> %a) {
;
; SSSE3-FAST-LABEL: PR37890_v8i16:
; SSSE3-FAST: # %bb.0:
-; SSSE3-FAST-NEXT: phaddw %xmm0, %xmm0
-; SSSE3-FAST-NEXT: phaddw %xmm0, %xmm0
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSSE3-FAST-NEXT: paddw %xmm0, %xmm1
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSSE3-FAST-NEXT: paddw %xmm1, %xmm0
; SSSE3-FAST-NEXT: phaddw %xmm0, %xmm0
; SSSE3-FAST-NEXT: movd %xmm0, %eax
; SSSE3-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -117,8 +121,10 @@ define i16 @PR37890_v8i16(<8 x i16> %a) {
;
; AVX1-FAST-LABEL: PR37890_v8i16:
; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -171,9 +177,10 @@ define i32 @PR37890_v8i32(<8 x i32> %a) {
; SSSE3-FAST-LABEL: PR37890_v8i32:
; SSSE3-FAST: # %bb.0:
; SSSE3-FAST-NEXT: paddd %xmm1, %xmm0
-; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm0
-; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm0
-; SSSE3-FAST-NEXT: movd %xmm0, %eax
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSSE3-FAST-NEXT: paddd %xmm0, %xmm1
+; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm1
+; SSSE3-FAST-NEXT: movd %xmm1, %eax
; SSSE3-FAST-NEXT: retq
;
; AVX1-SLOW-LABEL: PR37890_v8i32:
@@ -191,8 +198,9 @@ define i32 @PR37890_v8i32(<8 x i32> %a) {
; AVX1-FAST-LABEL: PR37890_v8i32:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: vzeroupper
@@ -253,8 +261,10 @@ define i16 @PR37890_v16i16(<16 x i16> %a) {
; SSSE3-FAST-LABEL: PR37890_v16i16:
; SSSE3-FAST: # %bb.0:
; SSSE3-FAST-NEXT: paddw %xmm1, %xmm0
-; SSSE3-FAST-NEXT: phaddw %xmm0, %xmm0
-; SSSE3-FAST-NEXT: phaddw %xmm0, %xmm0
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSSE3-FAST-NEXT: paddw %xmm0, %xmm1
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSSE3-FAST-NEXT: paddw %xmm1, %xmm0
; SSSE3-FAST-NEXT: phaddw %xmm0, %xmm0
; SSSE3-FAST-NEXT: movd %xmm0, %eax
; SSSE3-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -278,9 +288,11 @@ define i16 @PR37890_v16i16(<16 x i16> %a) {
; AVX1-FAST-LABEL: PR37890_v16i16:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm1, %xmm0
-; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -369,12 +381,13 @@ define i32 @PR37890_v16i32(<16 x i32> %a) {
;
; AVX1-FAST-LABEL: PR37890_v16i32:
; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm2
-; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
-; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm2
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm3
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm2, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: vzeroupper
diff --git a/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll b/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll
index 15cbd013981eb..c53496fd626df 100644
--- a/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll
+++ b/llvm/test/CodeGen/X86/horizontal-reduce-fadd.ll
@@ -30,7 +30,9 @@ define float @PR37890_v4f32(<4 x float> %a) {
;
; SSSE3-FAST-LABEL: PR37890_v4f32:
; SSSE3-FAST: # %bb.0:
-; SSSE3-FAST-NEXT: haddps %xmm0, %xmm0
+; SSSE3-FAST-NEXT: movaps %xmm0, %xmm1
+; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSSE3-FAST-NEXT: addps %xmm1, %xmm0
; SSSE3-FAST-NEXT: haddps %xmm0, %xmm0
; SSSE3-FAST-NEXT: retq
;
@@ -44,7 +46,8 @@ define float @PR37890_v4f32(<4 x float> %a) {
;
; AVX1-FAST-LABEL: PR37890_v4f32:
; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: retq
;
@@ -99,7 +102,7 @@ define double @PR37890_v4f64(<4 x double> %a) {
; AVX1-FAST-LABEL: PR37890_v4f64:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vhaddpd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddpd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
; AVX1-FAST-NEXT: retq
@@ -146,7 +149,9 @@ define float @PR37890_v8f32(<8 x float> %a) {
; SSSE3-FAST-LABEL: PR37890_v8f32:
; SSSE3-FAST: # %bb.0:
; SSSE3-FAST-NEXT: addps %xmm1, %xmm0
-; SSSE3-FAST-NEXT: haddps %xmm0, %xmm0
+; SSSE3-FAST-NEXT: movaps %xmm0, %xmm1
+; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
+; SSSE3-FAST-NEXT: addps %xmm1, %xmm0
; SSSE3-FAST-NEXT: haddps %xmm0, %xmm0
; SSSE3-FAST-NEXT: retq
;
@@ -164,8 +169,9 @@ define float @PR37890_v8f32(<8 x float> %a) {
; AVX1-FAST-LABEL: PR37890_v8f32:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm1, %xmm0
-; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
; AVX1-FAST-NEXT: retq
@@ -235,7 +241,7 @@ define double @PR37890_v8f64(<8 x double> %a) {
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vhaddpd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddpd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
; AVX1-FAST-NEXT: retq
@@ -314,8 +320,9 @@ define float @PR37890_v16f32(<16 x float> %a) {
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vaddps %ymm1, %ymm0, %ymm0
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm1, %xmm0
-; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
+; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
; AVX1-FAST-NEXT: retq
diff --git a/llvm/test/CodeGen/X86/horizontal-sum.ll b/llvm/test/CodeGen/X86/horizontal-sum.ll
index 568150cfa3971..3718a1381e4df 100644
--- a/llvm/test/CodeGen/X86/horizontal-sum.ll
+++ b/llvm/test/CodeGen/X86/horizontal-sum.ll
@@ -942,77 +942,62 @@ define <4 x float> @reduction_sum_v4f32_v4f32_reassoc(<4 x float> %0, <4 x float
; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
; SSSE3-SLOW-NEXT: addps %xmm4, %xmm0
; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
-; SSSE3-SLOW-NEXT: movaps %xmm1, %xmm5
-; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm1[1]
-; SSSE3-SLOW-NEXT: addps %xmm1, %xmm5
-; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm5[1,1,3,3]
-; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; SSSE3-SLOW-NEXT: addss %xmm4, %xmm0
+; SSSE3-SLOW-NEXT: movaps %xmm1, %xmm4
+; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]
+; SSSE3-SLOW-NEXT: addps %xmm1, %xmm4
+; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm4[1,1,3,3]
+; SSSE3-SLOW-NEXT: addss %xmm4, %xmm1
+; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm1
; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
; SSSE3-SLOW-NEXT: addps %xmm2, %xmm1
-; SSSE3-SLOW-NEXT: movaps %xmm3, %xmm2
-; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1]
-; SSSE3-SLOW-NEXT: addps %xmm3, %xmm2
-; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm3
-; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm3 = xmm3[0],xmm1[0]
-; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
-; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
-; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
-; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm4 = xmm4[0,1],xmm2[2,0]
-; SSSE3-SLOW-NEXT: addps %xmm4, %xmm0
+; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; SSSE3-SLOW-NEXT: addss %xmm1, %xmm2
+; SSSE3-SLOW-NEXT: movaps %xmm3, %xmm1
+; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm3[1]
+; SSSE3-SLOW-NEXT: addps %xmm3, %xmm1
+; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm3 = xmm1[1,1,3,3]
+; SSSE3-SLOW-NEXT: addss %xmm1, %xmm3
+; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
; SSSE3-SLOW-NEXT: retq
;
; SSSE3-FAST-LABEL: reduction_sum_v4f32_v4f32_reassoc:
; SSSE3-FAST: # %bb.0:
-; SSSE3-FAST-NEXT: movaps %xmm0, %xmm4
-; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm0[1]
-; SSSE3-FAST-NEXT: addps %xmm4, %xmm0
-; SSSE3-FAST-NEXT: movaps %xmm1, %xmm4
-; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm1[1]
-; SSSE3-FAST-NEXT: addps %xmm1, %xmm4
-; SSSE3-FAST-NEXT: haddps %xmm4, %xmm0
-; SSSE3-FAST-NEXT: movaps %xmm2, %xmm1
-; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm2[1]
-; SSSE3-FAST-NEXT: addps %xmm2, %xmm1
-; SSSE3-FAST-NEXT: movaps %xmm3, %xmm2
-; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm2 = xmm2[1],xmm3[1]
-; SSSE3-FAST-NEXT: addps %xmm3, %xmm2
-; SSSE3-FAST-NEXT: haddps %xmm2, %xmm1
-; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; SSSE3-FAST-NEXT: haddps %xmm3, %xmm2
+; SSSE3-FAST-NEXT: haddps %xmm1, %xmm0
+; SSSE3-FAST-NEXT: haddps %xmm2, %xmm0
; SSSE3-FAST-NEXT: retq
;
; AVX-SLOW-LABEL: reduction_sum_v4f32_v4f32_reassoc:
; AVX-SLOW: # %bb.0:
; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm4 = xmm0[1,0]
; AVX-SLOW-NEXT: vaddps %xmm4, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; AVX-SLOW-NEXT: vaddss %xmm4, %xmm0, %xmm0
; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm4 = xmm1[1,0]
; AVX-SLOW-NEXT: vaddps %xmm4, %xmm1, %xmm1
-; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm4 = xmm2[1,0]
-; AVX-SLOW-NEXT: vaddps %xmm4, %xmm2, %xmm2
-; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm4 = xmm3[1,0]
-; AVX-SLOW-NEXT: vaddps %xmm4, %xmm3, %xmm3
-; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm4 = xmm3[1,1],xmm2[1,1]
-; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm5 = xmm0[1],xmm1[1],zero,zero
-; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm4 = xmm5[0,1],xmm4[2,0]
-; AVX-SLOW-NEXT: vmovlhps {{.*#+}} xmm2 = xmm3[0],xmm2[0]
-; AVX-SLOW-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
-; AVX-SLOW-NEXT: vaddps %xmm4, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; AVX-SLOW-NEXT: vaddss %xmm4, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm1 = xmm2[1,0]
+; AVX-SLOW-NEXT: vaddps %xmm1, %xmm2, %xmm1
+; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX-SLOW-NEXT: vaddss %xmm2, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
+; AVX-SLOW-NEXT: vshufpd {{.*#+}} xmm1 = xmm3[1,0]
+; AVX-SLOW-NEXT: vaddps %xmm1, %xmm3, %xmm1
+; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX-SLOW-NEXT: vaddss %xmm2, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
; AVX-SLOW-NEXT: retq
;
; AVX-FAST-LABEL: reduction_sum_v4f32_v4f32_reassoc:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm4 = xmm0[1,0]
-; AVX-FAST-NEXT: vaddps %xmm4, %xmm0, %xmm0
-; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm4 = xmm1[1,0]
-; AVX-FAST-NEXT: vaddps %xmm4, %xmm1, %xmm1
+; AVX-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm2
; AVX-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0
-; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm1 = xmm2[1,0]
-; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm1
-; AVX-FAST-NEXT: vshufpd {{.*#+}} xmm2 = xmm3[1,0]
-; AVX-FAST-NEXT: vaddps %xmm2, %xmm3, %xmm2
-; AVX-FAST-NEXT: vhaddps %xmm2, %xmm1, %xmm1
-; AVX-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX-FAST-NEXT: vhaddps %xmm2, %xmm0, %xmm0
; AVX-FAST-NEXT: retq
%5 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %0)
%6 = call reassoc float @llvm.vector.reduce.fadd.f32.v4f32(float -0.0, <4 x float> %1)
@@ -1042,87 +1027,52 @@ define <4 x i32> @reduction_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32
; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm6
; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm3 = xmm6[1,1,1,1]
; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
+; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm1
; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
-; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm0
+; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSSE3-SLOW-NEXT: retq
;
; SSSE3-FAST-LABEL: reduction_sum_v4i32_v4i32:
; SSSE3-FAST: # %bb.0:
-; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; SSSE3-FAST-NEXT: paddd %xmm4, %xmm0
-; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; SSSE3-FAST-NEXT: paddd %xmm1, %xmm4
-; SSSE3-FAST-NEXT: phaddd %xmm4, %xmm0
-; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; SSSE3-FAST-NEXT: paddd %xmm2, %xmm1
-; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; SSSE3-FAST-NEXT: paddd %xmm3, %xmm2
-; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm1
-; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm2
+; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm0
+; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm0
; SSSE3-FAST-NEXT: retq
;
-; AVX1-SLOW-LABEL: reduction_sum_v4i32_v4i32:
-; AVX1-SLOW: # %bb.0:
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
-; AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm1, %xmm1
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
-; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
-; AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm2, %xmm2
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
-; AVX1-SLOW-NEXT: vpaddd %xmm6, %xmm3, %xmm3
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm2, %xmm2
-; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX1-SLOW-NEXT: retq
+; AVX-SLOW-LABEL: reduction_sum_v4i32_v4i32:
+; AVX-SLOW: # %bb.0:
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vmovd %xmm1, %eax
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; AVX-SLOW-NEXT: vpaddd %xmm1, %xmm2, %xmm1
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vmovd %xmm1, %ecx
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
+; AVX-SLOW-NEXT: vpaddd %xmm1, %xmm3, %xmm1
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vmovd %xmm1, %edx
+; AVX-SLOW-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX-SLOW-NEXT: retq
;
; AVX-FAST-LABEL: reduction_sum_v4i32_v4i32:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; AVX-FAST-NEXT: vpaddd %xmm4, %xmm0, %xmm0
-; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; AVX-FAST-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm2
; AVX-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0
-; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; AVX-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1
-; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; AVX-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
-; AVX-FAST-NEXT: vphaddd %xmm2, %xmm1, %xmm1
-; AVX-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX-FAST-NEXT: vphaddd %xmm2, %xmm0, %xmm0
; AVX-FAST-NEXT: retq
-;
-; AVX2-SLOW-LABEL: reduction_sum_v4i32_v4i32:
-; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
-; AVX2-SLOW-NEXT: vpaddd %xmm5, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
-; AVX2-SLOW-NEXT: vpaddd %xmm5, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
-; AVX2-SLOW-NEXT: vpaddd %xmm5, %xmm3, %xmm3
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0],xmm1[1],xmm4[2,3]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3]
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX2-SLOW-NEXT: vpbroadcastd %xmm3, %xmm1
-; AVX2-SLOW-NEXT: vpbroadcastd %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: retq
%5 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %0)
%6 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %1)
%7 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %2)
diff --git a/llvm/test/CodeGen/X86/insertelement-zero.ll b/llvm/test/CodeGen/X86/insertelement-zero.ll
index e1c8cefa73d8a..c012a908bebc9 100644
--- a/llvm/test/CodeGen/X86/insertelement-zero.ll
+++ b/llvm/test/CodeGen/X86/insertelement-zero.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=SSE,SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse3 | FileCheck %s --check-prefixes=SSE,SSE3
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
define <2 x double> @insert_v2f64_z1(<2 x double> %a) {
; SSE2-LABEL: insert_v2f64_z1:
@@ -556,22 +556,17 @@ define <4 x i32> @PR41512_loads(ptr %p1, ptr %p2) {
define i64 @fold_insertelement_to_and(i32 noundef %arg) {
; SSE-LABEL: fold_insertelement_to_and:
; SSE: # %bb.0:
-; SSE-NEXT: movl $154, %eax
+; SSE-NEXT: pxor %xmm0, %xmm0
+; SSE-NEXT: psadbw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; SSE-NEXT: movq %xmm0, %rax
; SSE-NEXT: retq
;
-; AVX1-LABEL: fold_insertelement_to_and:
-; AVX1: # %bb.0:
-; AVX1-NEXT: movl $154, %eax
-; AVX1-NEXT: retq
-;
-; AVX2-LABEL: fold_insertelement_to_and:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpmovsxbq {{.*#+}} xmm0 = [0,77]
-; AVX2-NEXT: vpaddq %xmm0, %xmm0, %xmm1
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: retq
+; AVX-LABEL: fold_insertelement_to_and:
+; AVX: # %bb.0:
+; AVX-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX-NEXT: vpsadbw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: retq
%i = shufflevector <8 x i64> zeroinitializer, <8 x i64> splat (i64 77), <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 8, i32 6, i32 10>
%i1 = insertelement <8 x i64> %i, i64 0, i64 0
%i2 = insertelement <8 x i64> %i1, i64 0, i64 2
@@ -579,3 +574,5 @@ define i64 @fold_insertelement_to_and(i32 noundef %arg) {
%i4 = tail call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %i3)
ret i64 %i4
}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 201a0900e6eb9..fc7b176d07400 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -292,7 +292,7 @@ define i32 @_Z10test_shortPsS_i_512(ptr nocapture readonly, ptr nocapture readon
; AVX512-NEXT: jne .LBB2_1
; AVX512-NEXT: # %bb.2: # %middle.block
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -334,10 +334,10 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
; SSE2-NEXT: movl %edx, %eax
; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: xorl %ecx, %ecx
-; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: pxor %xmm4, %xmm4
-; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB3_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
@@ -347,27 +347,27 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
; SSE2-NEXT: movdqu 48(%rdi,%rcx,2), %xmm8
; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm9
; SSE2-NEXT: pmaddwd %xmm5, %xmm9
-; SSE2-NEXT: paddd %xmm9, %xmm2
+; SSE2-NEXT: paddd %xmm9, %xmm4
; SSE2-NEXT: movdqu 16(%rsi,%rcx,2), %xmm5
; SSE2-NEXT: pmaddwd %xmm6, %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm4
+; SSE2-NEXT: paddd %xmm5, %xmm3
; SSE2-NEXT: movdqu 32(%rsi,%rcx,2), %xmm5
; SSE2-NEXT: pmaddwd %xmm7, %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm1
+; SSE2-NEXT: paddd %xmm5, %xmm2
; SSE2-NEXT: movdqu 48(%rsi,%rcx,2), %xmm5
; SSE2-NEXT: pmaddwd %xmm8, %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm3
+; SSE2-NEXT: paddd %xmm5, %xmm1
; SSE2-NEXT: addq $16, %rcx
; SSE2-NEXT: cmpq %rcx, %rax
; SSE2-NEXT: jne .LBB3_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm0, %xmm4
-; SSE2-NEXT: paddd %xmm0, %xmm3
-; SSE2-NEXT: paddd %xmm4, %xmm3
; SSE2-NEXT: paddd %xmm0, %xmm2
+; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: paddd %xmm0, %xmm3
; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
; SSE2-NEXT: paddd %xmm3, %xmm1
+; SSE2-NEXT: paddd %xmm2, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; SSE2-NEXT: paddd %xmm1, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -405,16 +405,16 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
; AVX1-NEXT: cmpq %rcx, %rax
; AVX1-NEXT: jne .LBB3_1
; AVX1-NEXT: # %bb.2: # %middle.block
-; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm3
-; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm4
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vpaddd %xmm4, %xmm5, %xmm4
; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -477,7 +477,7 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
; AVX512F-NEXT: # %bb.2: # %middle.block
; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -506,7 +506,7 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -848,7 +848,7 @@ define i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocapture readonly
; AVX512-NEXT: jne .LBB6_1
; AVX512-NEXT: # %bb.2: # %middle.block
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -890,10 +890,10 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
; SSE2-NEXT: movl %edx, %eax
; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: xorl %ecx, %ecx
-; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: pxor %xmm4, %xmm4
-; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB7_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
@@ -906,36 +906,36 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3],xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]
; SSE2-NEXT: psraw $8, %xmm10
; SSE2-NEXT: pmaddwd %xmm9, %xmm10
-; SSE2-NEXT: paddd %xmm10, %xmm2
+; SSE2-NEXT: paddd %xmm10, %xmm4
; SSE2-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; SSE2-NEXT: psraw $8, %xmm7
; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; SSE2-NEXT: psraw $8, %xmm8
; SSE2-NEXT: pmaddwd %xmm7, %xmm8
-; SSE2-NEXT: paddd %xmm8, %xmm4
+; SSE2-NEXT: paddd %xmm8, %xmm3
; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
; SSE2-NEXT: psraw $8, %xmm7
; SSE2-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3],xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]
; SSE2-NEXT: psraw $8, %xmm8
; SSE2-NEXT: pmaddwd %xmm7, %xmm8
-; SSE2-NEXT: paddd %xmm8, %xmm1
+; SSE2-NEXT: paddd %xmm8, %xmm2
; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; SSE2-NEXT: psraw $8, %xmm6
; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; SSE2-NEXT: psraw $8, %xmm5
; SSE2-NEXT: pmaddwd %xmm6, %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm3
+; SSE2-NEXT: paddd %xmm5, %xmm1
; SSE2-NEXT: addq $32, %rcx
; SSE2-NEXT: cmpq %rcx, %rax
; SSE2-NEXT: jne .LBB7_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm0, %xmm4
-; SSE2-NEXT: paddd %xmm0, %xmm3
-; SSE2-NEXT: paddd %xmm4, %xmm3
; SSE2-NEXT: paddd %xmm0, %xmm2
+; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: paddd %xmm0, %xmm3
; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
; SSE2-NEXT: paddd %xmm3, %xmm1
+; SSE2-NEXT: paddd %xmm2, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; SSE2-NEXT: paddd %xmm1, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -977,16 +977,16 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
; AVX1-NEXT: cmpq %rcx, %rax
; AVX1-NEXT: jne .LBB7_1
; AVX1-NEXT: # %bb.2: # %middle.block
-; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm3
-; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm4
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vpaddd %xmm4, %xmm5, %xmm4
; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1053,7 +1053,7 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
; AVX512F-NEXT: # %bb.2: # %middle.block
; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1083,7 +1083,7 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1450,7 +1450,7 @@ define i32 @test_unsigned_short_512(ptr nocapture readonly, ptr nocapture readon
; AVX512-NEXT: jne .LBB10_1
; AVX512-NEXT: # %bb.2: # %middle.block
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1490,15 +1490,15 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
; SSE2-LABEL: test_unsigned_short_1024:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: movl %edx, %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: xorl %ecx, %ecx
; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pxor %xmm7, %xmm7
+; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB11_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
@@ -1509,27 +1509,27 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
; SSE2-NEXT: pmullw %xmm8, %xmm9
; SSE2-NEXT: movdqa %xmm9, %xmm8
; SSE2-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm8, %xmm7
+; SSE2-NEXT: paddd %xmm8, %xmm5
; SSE2-NEXT: movdqu 32(%rdi,%rcx,2), %xmm8
; SSE2-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
; SSE2-NEXT: movdqu 32(%rsi,%rcx,2), %xmm10
-; SSE2-NEXT: paddd %xmm9, %xmm5
+; SSE2-NEXT: paddd %xmm9, %xmm7
; SSE2-NEXT: movdqa %xmm10, %xmm9
; SSE2-NEXT: pmulhuw %xmm8, %xmm9
; SSE2-NEXT: pmullw %xmm8, %xmm10
; SSE2-NEXT: movdqa %xmm10, %xmm8
; SSE2-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm9[4],xmm8[5],xmm9[5],xmm8[6],xmm9[6],xmm8[7],xmm9[7]
-; SSE2-NEXT: paddd %xmm8, %xmm6
+; SSE2-NEXT: paddd %xmm8, %xmm4
; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm8
; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm9
-; SSE2-NEXT: paddd %xmm10, %xmm4
+; SSE2-NEXT: paddd %xmm10, %xmm6
; SSE2-NEXT: movdqa %xmm9, %xmm10
; SSE2-NEXT: pmulhuw %xmm8, %xmm10
; SSE2-NEXT: pmullw %xmm8, %xmm9
; SSE2-NEXT: movdqa %xmm9, %xmm8
; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm8, %xmm0
+; SSE2-NEXT: paddd %xmm8, %xmm2
; SSE2-NEXT: movdqu 16(%rdi,%rcx,2), %xmm8
; SSE2-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]
; SSE2-NEXT: movdqu 16(%rsi,%rcx,2), %xmm10
@@ -1541,23 +1541,23 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
; SSE2-NEXT: paddd %xmm8, %xmm1
; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
-; SSE2-NEXT: paddd %xmm10, %xmm2
+; SSE2-NEXT: paddd %xmm10, %xmm0
; SSE2-NEXT: addq $16, %rcx
; SSE2-NEXT: cmpq %rcx, %rax
; SSE2-NEXT: jne .LBB11_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm6, %xmm3
-; SSE2-NEXT: paddd %xmm7, %xmm2
-; SSE2-NEXT: paddd %xmm3, %xmm2
-; SSE2-NEXT: paddd %xmm4, %xmm0
-; SSE2-NEXT: paddd %xmm5, %xmm1
-; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: paddd %xmm6, %xmm2
+; SSE2-NEXT: paddd %xmm7, %xmm1
; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: paddd %xmm4, %xmm3
+; SSE2-NEXT: paddd %xmm5, %xmm0
+; SSE2-NEXT: paddd %xmm3, %xmm0
; SSE2-NEXT: paddd %xmm1, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: retq
;
; AVX1-LABEL: test_unsigned_short_1024:
@@ -1615,17 +1615,17 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
; AVX1-NEXT: cmpq %rcx, %rax
; AVX1-NEXT: jne .LBB11_1
; AVX1-NEXT: # %bb.2: # %middle.block
-; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm4
-; AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm5
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm5
+; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6
+; AVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5
; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
; AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm1
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpaddd %xmm0, %xmm4, %xmm0
+; AVX1-NEXT: vpaddd %xmm4, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1701,7 +1701,7 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
; AVX512-NEXT: # %bb.2: # %middle.block
; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -3106,7 +3106,7 @@ define i32 @add_used_by_loop_phi(ptr %a, ptr %b, i64 %offset_a, i64 %offset_b, i
; AVX512-NEXT: jb .LBB38_1
; AVX512-NEXT: # %bb.2: # %afterloop
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index 4c89fa95fe4b2..e1acb933ed2a1 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -320,7 +320,7 @@ define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocaptur
; CHECK-SKX-NEXT: # %bb.2: # %middle.block
; CHECK-SKX-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; CHECK-SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-SKX-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-SKX-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-SKX-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -350,7 +350,7 @@ define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocaptur
; CHECK-AVX512-NEXT: # %bb.2: # %middle.block
; CHECK-AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; CHECK-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -380,7 +380,7 @@ define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocaptur
; CHECK-VBMI-NEXT: # %bb.2: # %middle.block
; CHECK-VBMI-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; CHECK-VBMI-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-VBMI-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-VBMI-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -537,7 +537,7 @@ define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {
; CHECK-SKX-NEXT: jne .LBB11_1
; CHECK-SKX-NEXT: # %bb.2: # %middle.block
; CHECK-SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-SKX-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-SKX-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-SKX-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -562,7 +562,7 @@ define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {
; CHECK-AVX512-NEXT: jne .LBB11_1
; CHECK-AVX512-NEXT: # %bb.2: # %middle.block
; CHECK-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -587,7 +587,7 @@ define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {
; CHECK-VBMI-NEXT: jne .LBB11_1
; CHECK-VBMI-NEXT: # %bb.2: # %middle.block
; CHECK-VBMI-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-VBMI-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-VBMI-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/phaddsub-extract.ll b/llvm/test/CodeGen/X86/phaddsub-extract.ll
index b38a10c7e4263..b9b07c22bb034 100644
--- a/llvm/test/CodeGen/X86/phaddsub-extract.ll
+++ b/llvm/test/CodeGen/X86/phaddsub-extract.ll
@@ -1699,7 +1699,8 @@ define i32 @partial_reduction_add_v8i32(<8 x i32> %x) {
;
; AVX-FAST-LABEL: partial_reduction_add_v8i32:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX-FAST-NEXT: vmovd %xmm0, %eax
; AVX-FAST-NEXT: vzeroupper
@@ -1740,13 +1741,33 @@ define i32 @partial_reduction_add_v16i32(<16 x i32> %x) {
; AVX-SLOW-NEXT: vzeroupper
; AVX-SLOW-NEXT: retq
;
-; AVX-FAST-LABEL: partial_reduction_add_v16i32:
-; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX-FAST-NEXT: vmovd %xmm0, %eax
-; AVX-FAST-NEXT: vzeroupper
-; AVX-FAST-NEXT: retq
+; AVX1-FAST-LABEL: partial_reduction_add_v16i32:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; AVX1-FAST-NEXT: vzeroupper
+; AVX1-FAST-NEXT: retq
+;
+; AVX2-FAST-LABEL: partial_reduction_add_v16i32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vmovd %xmm0, %eax
+; AVX2-FAST-NEXT: vzeroupper
+; AVX2-FAST-NEXT: retq
+;
+; AVX512-FAST-LABEL: partial_reduction_add_v16i32:
+; AVX512-FAST: # %bb.0:
+; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vmovd %xmm0, %eax
+; AVX512-FAST-NEXT: vzeroupper
+; AVX512-FAST-NEXT: retq
%x23 = shufflevector <16 x i32> %x, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%x0213 = add <16 x i32> %x, %x23
%x13 = shufflevector <16 x i32> %x0213, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1937,8 +1958,10 @@ define i16 @hadd16_8(<8 x i16> %x223) {
;
; SSE3-FAST-LABEL: hadd16_8:
; SSE3-FAST: # %bb.0:
-; SSE3-FAST-NEXT: phaddw %xmm0, %xmm0
-; SSE3-FAST-NEXT: phaddw %xmm0, %xmm0
+; SSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE3-FAST-NEXT: paddw %xmm0, %xmm1
+; SSE3-FAST-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE3-FAST-NEXT: paddw %xmm1, %xmm0
; SSE3-FAST-NEXT: phaddw %xmm0, %xmm0
; SSE3-FAST-NEXT: movd %xmm0, %eax
; SSE3-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1958,8 +1981,10 @@ define i16 @hadd16_8(<8 x i16> %x223) {
;
; AVX-FAST-LABEL: hadd16_8:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
-; AVX-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX-FAST-NEXT: vmovd %xmm0, %eax
; AVX-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1986,9 +2011,10 @@ define i32 @hadd32_4(<4 x i32> %x225) {
;
; SSE3-FAST-LABEL: hadd32_4:
; SSE3-FAST: # %bb.0:
-; SSE3-FAST-NEXT: phaddd %xmm0, %xmm0
-; SSE3-FAST-NEXT: phaddd %xmm0, %xmm0
-; SSE3-FAST-NEXT: movd %xmm0, %eax
+; SSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE3-FAST-NEXT: paddd %xmm0, %xmm1
+; SSE3-FAST-NEXT: phaddd %xmm1, %xmm1
+; SSE3-FAST-NEXT: movd %xmm1, %eax
; SSE3-FAST-NEXT: retq
;
; AVX-SLOW-LABEL: hadd32_4:
@@ -2002,7 +2028,8 @@ define i32 @hadd32_4(<4 x i32> %x225) {
;
; AVX-FAST-LABEL: hadd32_4:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX-FAST-NEXT: vmovd %xmm0, %eax
; AVX-FAST-NEXT: retq
@@ -2044,7 +2071,8 @@ define i32 @hadd32_8(<8 x i32> %x225) {
;
; AVX-FAST-LABEL: hadd32_8:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX-FAST-NEXT: vmovd %xmm0, %eax
; AVX-FAST-NEXT: vzeroupper
@@ -2085,13 +2113,33 @@ define i32 @hadd32_16(<16 x i32> %x225) {
; AVX-SLOW-NEXT: vzeroupper
; AVX-SLOW-NEXT: retq
;
-; AVX-FAST-LABEL: hadd32_16:
-; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX-FAST-NEXT: vmovd %xmm0, %eax
-; AVX-FAST-NEXT: vzeroupper
-; AVX-FAST-NEXT: retq
+; AVX1-FAST-LABEL: hadd32_16:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; AVX1-FAST-NEXT: vzeroupper
+; AVX1-FAST-NEXT: retq
+;
+; AVX2-FAST-LABEL: hadd32_16:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vmovd %xmm0, %eax
+; AVX2-FAST-NEXT: vzeroupper
+; AVX2-FAST-NEXT: retq
+;
+; AVX512-FAST-LABEL: hadd32_16:
+; AVX512-FAST: # %bb.0:
+; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vmovd %xmm0, %eax
+; AVX512-FAST-NEXT: vzeroupper
+; AVX512-FAST-NEXT: retq
%x226 = shufflevector <16 x i32> %x225, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%x227 = add <16 x i32> %x225, %x226
%x228 = shufflevector <16 x i32> %x227, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -2103,8 +2151,10 @@ define i32 @hadd32_16(<16 x i32> %x225) {
define i16 @hadd16_8_optsize(<8 x i16> %x223) optsize {
; SSE3-LABEL: hadd16_8_optsize:
; SSE3: # %bb.0:
-; SSE3-NEXT: phaddw %xmm0, %xmm0
-; SSE3-NEXT: phaddw %xmm0, %xmm0
+; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE3-NEXT: paddw %xmm0, %xmm1
+; SSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE3-NEXT: paddw %xmm1, %xmm0
; SSE3-NEXT: phaddw %xmm0, %xmm0
; SSE3-NEXT: movd %xmm0, %eax
; SSE3-NEXT: # kill: def $ax killed $ax killed $eax
@@ -2112,8 +2162,10 @@ define i16 @hadd16_8_optsize(<8 x i16> %x223) optsize {
;
; AVX-LABEL: hadd16_8_optsize:
; AVX: # %bb.0:
-; AVX-NEXT: vphaddw %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX-NEXT: vmovd %xmm0, %eax
; AVX-NEXT: # kill: def $ax killed $ax killed $eax
@@ -2131,14 +2183,16 @@ define i16 @hadd16_8_optsize(<8 x i16> %x223) optsize {
define i32 @hadd32_4_optsize(<4 x i32> %x225) optsize {
; SSE3-LABEL: hadd32_4_optsize:
; SSE3: # %bb.0:
-; SSE3-NEXT: phaddd %xmm0, %xmm0
-; SSE3-NEXT: phaddd %xmm0, %xmm0
-; SSE3-NEXT: movd %xmm0, %eax
+; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE3-NEXT: paddd %xmm0, %xmm1
+; SSE3-NEXT: phaddd %xmm1, %xmm1
+; SSE3-NEXT: movd %xmm1, %eax
; SSE3-NEXT: retq
;
; AVX-LABEL: hadd32_4_optsize:
; AVX: # %bb.0:
-; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX-NEXT: vmovd %xmm0, %eax
; AVX-NEXT: retq
@@ -2153,14 +2207,16 @@ define i32 @hadd32_4_optsize(<4 x i32> %x225) optsize {
define i32 @hadd32_4_pgso(<4 x i32> %x225) !prof !14 {
; SSE3-LABEL: hadd32_4_pgso:
; SSE3: # %bb.0:
-; SSE3-NEXT: phaddd %xmm0, %xmm0
-; SSE3-NEXT: phaddd %xmm0, %xmm0
-; SSE3-NEXT: movd %xmm0, %eax
+; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE3-NEXT: paddd %xmm0, %xmm1
+; SSE3-NEXT: phaddd %xmm1, %xmm1
+; SSE3-NEXT: movd %xmm1, %eax
; SSE3-NEXT: retq
;
; AVX-LABEL: hadd32_4_pgso:
; AVX: # %bb.0:
-; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX-NEXT: vmovd %xmm0, %eax
; AVX-NEXT: retq
@@ -2183,7 +2239,8 @@ define i32 @hadd32_8_optsize(<8 x i32> %x225) optsize {
;
; AVX-LABEL: hadd32_8_optsize:
; AVX: # %bb.0:
-; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX-NEXT: vmovd %xmm0, %eax
; AVX-NEXT: vzeroupper
@@ -2205,13 +2262,61 @@ define i32 @hadd32_16_optsize(<16 x i32> %x225) optsize {
; SSE3-NEXT: movd %xmm1, %eax
; SSE3-NEXT: retq
;
-; AVX-LABEL: hadd32_16_optsize:
-; AVX: # %bb.0:
-; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vzeroupper
-; AVX-NEXT: retq
+; AVX1-SLOW-LABEL: hadd32_16_optsize:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT: vzeroupper
+; AVX1-SLOW-NEXT: retq
+;
+; AVX1-FAST-LABEL: hadd32_16_optsize:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; AVX1-FAST-NEXT: vzeroupper
+; AVX1-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: hadd32_16_optsize:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vmovd %xmm0, %eax
+; AVX2-SLOW-NEXT: vzeroupper
+; AVX2-SLOW-NEXT: retq
+;
+; AVX2-FAST-LABEL: hadd32_16_optsize:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vmovd %xmm0, %eax
+; AVX2-FAST-NEXT: vzeroupper
+; AVX2-FAST-NEXT: retq
+;
+; AVX512-SLOW-LABEL: hadd32_16_optsize:
+; AVX512-SLOW: # %bb.0:
+; AVX512-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-SLOW-NEXT: vmovd %xmm0, %eax
+; AVX512-SLOW-NEXT: vzeroupper
+; AVX512-SLOW-NEXT: retq
+;
+; AVX512-FAST-LABEL: hadd32_16_optsize:
+; AVX512-FAST: # %bb.0:
+; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vmovd %xmm0, %eax
+; AVX512-FAST-NEXT: vzeroupper
+; AVX512-FAST-NEXT: retq
%x226 = shufflevector <16 x i32> %x225, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%x227 = add <16 x i32> %x225, %x226
%x228 = shufflevector <16 x i32> %x227, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
diff --git a/llvm/test/CodeGen/X86/pr173924.ll b/llvm/test/CodeGen/X86/pr173924.ll
index d86b82edb7249..9db18894f8311 100644
--- a/llvm/test/CodeGen/X86/pr173924.ll
+++ b/llvm/test/CodeGen/X86/pr173924.ll
@@ -1,36 +1,82 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s
-; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=AVX2
+; RUN: llc < %s -mtriple=x86_64-- -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512
define i256 @PR173924(<8 x i256> %a0) {
-; CHECK-LABEL: PR173924:
-; CHECK: # %bb.0:
-; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %ecx
-; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %edi
-; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %r8d
-; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %r10d
-; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %r11d
-; CHECK-NEXT: andl $1, %r11d
-; CHECK-NEXT: andl $1, %r9d
-; CHECK-NEXT: addl %r11d, %r9d
-; CHECK-NEXT: andl $1, %r10d
-; CHECK-NEXT: andl $1, %edx
-; CHECK-NEXT: addl %r10d, %edx
-; CHECK-NEXT: addl %r9d, %edx
-; CHECK-NEXT: andl $1, %r8d
-; CHECK-NEXT: andl $1, %esi
-; CHECK-NEXT: addl %r8d, %esi
-; CHECK-NEXT: andl $1, %edi
-; CHECK-NEXT: andl $1, %ecx
-; CHECK-NEXT: addl %edi, %ecx
-; CHECK-NEXT: addl %esi, %ecx
-; CHECK-NEXT: addl %edx, %ecx
-; CHECK-NEXT: vmovd %ecx, %xmm0
-; CHECK-NEXT: vmovdqu %ymm0, (%rax)
-; CHECK-NEXT: vzeroupper
-; CHECK-NEXT: retq
+; AVX2-LABEL: PR173924:
+; AVX2: # %bb.0:
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT: andl $1, %r9d
+; AVX2-NEXT: andl $1, %esi
+; AVX2-NEXT: andb $1, %al
+; AVX2-NEXT: vmovd %esi, %xmm0
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $1, %r9d, %xmm0, %xmm0
+; AVX2-NEXT: vpinsrb $2, %eax, %xmm0, %xmm0
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT: andb $1, %al
+; AVX2-NEXT: vpinsrb $3, %eax, %xmm0, %xmm0
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT: andb $1, %al
+; AVX2-NEXT: vpinsrb $4, %eax, %xmm0, %xmm0
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT: andb $1, %al
+; AVX2-NEXT: vpinsrb $5, %eax, %xmm0, %xmm0
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT: andb $1, %al
+; AVX2-NEXT: vpinsrb $6, %eax, %xmm0, %xmm0
+; AVX2-NEXT: movzbl {{[0-9]+}}(%rsp), %eax
+; AVX2-NEXT: andb $1, %al
+; AVX2-NEXT: vpinsrb $7, %eax, %xmm0, %xmm0
+; AVX2-NEXT: movq %rdi, %rax
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vmovdqu %xmm1, 8(%rdi)
+; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovq %xmm0, (%rdi)
+; AVX2-NEXT: movq $0, 24(%rdi)
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: PR173924:
+; AVX512: # %bb.0:
+; AVX512-NEXT: movq %rdi, %rax
+; AVX512-NEXT: movl {{[0-9]+}}(%rsp), %edi
+; AVX512-NEXT: movl {{[0-9]+}}(%rsp), %r8d
+; AVX512-NEXT: movl {{[0-9]+}}(%rsp), %r10d
+; AVX512-NEXT: movl {{[0-9]+}}(%rsp), %r11d
+; AVX512-NEXT: movl {{[0-9]+}}(%rsp), %edx
+; AVX512-NEXT: movl {{[0-9]+}}(%rsp), %ecx
+; AVX512-NEXT: andl $1, %ecx
+; AVX512-NEXT: andl $1, %edx
+; AVX512-NEXT: andl $1, %esi
+; AVX512-NEXT: andl $1, %r9d
+; AVX512-NEXT: andl $1, %r11d
+; AVX512-NEXT: andl $1, %r10d
+; AVX512-NEXT: andl $1, %r8d
+; AVX512-NEXT: andl $1, %edi
+; AVX512-NEXT: vmovd %edi, %xmm0
+; AVX512-NEXT: vmovd %r8d, %xmm1
+; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1],xmm1[2],xmm0[2],xmm1[3],xmm0[3],xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-NEXT: vpsllq $48, %xmm0, %xmm0
+; AVX512-NEXT: vmovd %r10d, %xmm1
+; AVX512-NEXT: vmovd %r11d, %xmm2
+; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512-NEXT: vpbroadcastw %xmm1, %xmm1
+; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2],xmm0[3],xmm1[4,5,6,7]
+; AVX512-NEXT: vmovd %r9d, %xmm1
+; AVX512-NEXT: vmovd %esi, %xmm2
+; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; AVX512-NEXT: vmovd %edx, %xmm2
+; AVX512-NEXT: vmovd %ecx, %xmm3
+; AVX512-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; AVX512-NEXT: vpslld $16, %xmm2, %xmm2
+; AVX512-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1],xmm1[2,3,4,5,6,7]
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm1[0],xmm0[1],xmm1[2,3]
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovdqu %xmm1, 8(%rax)
+; AVX512-NEXT: vmovq %xmm0, (%rax)
+; AVX512-NEXT: movq $0, 24(%rax)
+; AVX512-NEXT: retq
%m = and <8 x i256> %a0, splat (i256 1)
%r = call i256 @llvm.vector.reduce.add.v8i256(<8 x i256> %m)
ret i256 %r
diff --git a/llvm/test/CodeGen/X86/pr62286.ll b/llvm/test/CodeGen/X86/pr62286.ll
index 161e9651a9cf2..222a20936d375 100644
--- a/llvm/test/CodeGen/X86/pr62286.ll
+++ b/llvm/test/CodeGen/X86/pr62286.ll
@@ -8,17 +8,18 @@ define i64 @PR62286(i32 %a) {
; SSE-LABEL: PR62286:
; SSE: # %bb.0:
; SSE-NEXT: movd %edi, %xmm0
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,1,1,0]
-; SSE-NEXT: paddd %xmm1, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,0,1,0]
+; SSE-NEXT: paddd %xmm0, %xmm0
; SSE-NEXT: pxor %xmm2, %xmm2
-; SSE-NEXT: pxor %xmm3, %xmm3
-; SSE-NEXT: pcmpgtd %xmm1, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,0,1,0]
-; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: pcmpgtd %xmm0, %xmm2
; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT: pxor %xmm3, %xmm3
+; SSE-NEXT: pcmpgtd %xmm0, %xmm3
+; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE-NEXT: pcmpgtd %xmm1, %xmm2
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE-NEXT: paddq %xmm0, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; SSE-NEXT: paddq %xmm1, %xmm0
; SSE-NEXT: movq %xmm0, %rax
; SSE-NEXT: retq
@@ -73,7 +74,7 @@ define i64 @PR62286(i32 %a) {
; AVX512-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1}
; AVX512-NEXT: vpmovsxdq %ymm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/sad.ll b/llvm/test/CodeGen/X86/sad.ll
index 095c8cfa1717c..7a31c896cce9f 100644
--- a/llvm/test/CodeGen/X86/sad.ll
+++ b/llvm/test/CodeGen/X86/sad.ll
@@ -101,7 +101,7 @@ define dso_local i32 @sad_16i8() nounwind {
; AVX512-NEXT: jne .LBB0_1
; AVX512-NEXT: # %bb.2: # %middle.block
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -158,8 +158,8 @@ define dso_local i32 @sad_32i8() nounwind {
; SSE2-NEXT: addq $32, %rax
; SSE2-NEXT: jne .LBB1_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm0, %xmm2
; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm2
; SSE2-NEXT: paddd %xmm0, %xmm0
; SSE2-NEXT: paddd %xmm0, %xmm1
; SSE2-NEXT: paddd %xmm2, %xmm0
@@ -190,15 +190,15 @@ define dso_local i32 @sad_32i8() nounwind {
; AVX1-NEXT: addq $32, %rax
; AVX1-NEXT: jne .LBB1_1
; AVX1-NEXT: # %bb.2: # %middle.block
-; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT: vpaddd %xmm4, %xmm4, %xmm5
-; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT: vpaddd %xmm3, %xmm3, %xmm4
; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm3
; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -250,7 +250,7 @@ define dso_local i32 @sad_32i8() nounwind {
; AVX512-NEXT: # %bb.2: # %middle.block
; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -291,18 +291,18 @@ middle.block:
define dso_local i32 @sad_avx64i8() nounwind {
; SSE2-LABEL: sad_avx64i8:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
-; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB2_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
; SSE2-NEXT: movdqa a+1024(%rax), %xmm5
; SSE2-NEXT: psadbw b+1024(%rax), %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm0
+; SSE2-NEXT: paddd %xmm5, %xmm4
; SSE2-NEXT: movdqa a+1040(%rax), %xmm5
; SSE2-NEXT: psadbw b+1040(%rax), %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm3
@@ -311,28 +311,28 @@ define dso_local i32 @sad_avx64i8() nounwind {
; SSE2-NEXT: paddd %xmm5, %xmm2
; SSE2-NEXT: movdqa a+1072(%rax), %xmm5
; SSE2-NEXT: psadbw b+1072(%rax), %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm1
+; SSE2-NEXT: paddd %xmm5, %xmm0
; SSE2-NEXT: addq $64, %rax
; SSE2-NEXT: jne .LBB2_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: paddd %xmm1, %xmm4
; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm5
-; SSE2-NEXT: paddd %xmm4, %xmm0
-; SSE2-NEXT: paddd %xmm4, %xmm1
-; SSE2-NEXT: paddd %xmm4, %xmm3
+; SSE2-NEXT: paddd %xmm5, %xmm4
+; SSE2-NEXT: paddd %xmm1, %xmm2
+; SSE2-NEXT: paddd %xmm5, %xmm2
+; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: paddd %xmm1, %xmm3
; SSE2-NEXT: paddd %xmm5, %xmm3
-; SSE2-NEXT: paddd %xmm5, %xmm1
-; SSE2-NEXT: paddd %xmm3, %xmm1
-; SSE2-NEXT: paddd %xmm5, %xmm0
-; SSE2-NEXT: paddd %xmm2, %xmm5
-; SSE2-NEXT: paddd %xmm0, %xmm5
-; SSE2-NEXT: paddd %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; SSE2-NEXT: paddd %xmm1, %xmm0
; SSE2-NEXT: paddd %xmm5, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: retq
;
; AVX1-LABEL: sad_avx64i8:
@@ -363,22 +363,22 @@ define dso_local i32 @sad_avx64i8() nounwind {
; AVX1-NEXT: addq $64, %rax
; AVX1-NEXT: jne .LBB2_1
; AVX1-NEXT: # %bb.2: # %middle.block
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
; AVX1-NEXT: vpaddd %xmm4, %xmm4, %xmm5
-; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6
-; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm7
-; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm8
-; AVX1-NEXT: vpaddd %xmm0, %xmm8, %xmm8
-; AVX1-NEXT: vpaddd %xmm2, %xmm8, %xmm2
-; AVX1-NEXT: vpaddd %xmm7, %xmm0, %xmm0
+; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm0, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm6, %xmm2
-; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -391,26 +391,26 @@ define dso_local i32 @sad_avx64i8() nounwind {
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB2_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
; AVX2-NEXT: vmovdqa a+1024(%rax), %ymm3
; AVX2-NEXT: vpsadbw b+1024(%rax), %ymm3, %ymm3
-; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vmovdqa a+1056(%rax), %ymm3
; AVX2-NEXT: vpsadbw b+1056(%rax), %ymm3, %ymm3
-; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
; AVX2-NEXT: addq $64, %rax
; AVX2-NEXT: jne .LBB2_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm2
; AVX2-NEXT: vpaddd %ymm0, %ymm0, %ymm3
+; AVX2-NEXT: vpaddd %ymm3, %ymm2, %ymm2
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vpaddd %ymm3, %ymm0, %ymm0
-; AVX2-NEXT: vpaddd %ymm3, %ymm2, %ymm1
-; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -442,7 +442,7 @@ define dso_local i32 @sad_avx64i8() nounwind {
; AVX512F-NEXT: vpaddd %zmm0, %zmm0, %zmm0
; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -471,7 +471,7 @@ define dso_local i32 @sad_avx64i8() nounwind {
; AVX512BW-NEXT: vpaddd %zmm0, %zmm0, %zmm0
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -513,18 +513,18 @@ middle.block:
define i32 @sad_avx64i8_prefer_256() nounwind "min-legal-vector-width"="256" "prefer-vector-width"="256" {
; SSE2-LABEL: sad_avx64i8_prefer_256:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
-; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB3_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
; SSE2-NEXT: movdqa a+1024(%rax), %xmm5
; SSE2-NEXT: psadbw b+1024(%rax), %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm0
+; SSE2-NEXT: paddd %xmm5, %xmm4
; SSE2-NEXT: movdqa a+1040(%rax), %xmm5
; SSE2-NEXT: psadbw b+1040(%rax), %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm3
@@ -533,28 +533,28 @@ define i32 @sad_avx64i8_prefer_256() nounwind "min-legal-vector-width"="256" "pr
; SSE2-NEXT: paddd %xmm5, %xmm2
; SSE2-NEXT: movdqa a+1072(%rax), %xmm5
; SSE2-NEXT: psadbw b+1072(%rax), %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm1
+; SSE2-NEXT: paddd %xmm5, %xmm0
; SSE2-NEXT: addq $64, %rax
; SSE2-NEXT: jne .LBB3_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: paddd %xmm1, %xmm4
; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm5
-; SSE2-NEXT: paddd %xmm4, %xmm0
-; SSE2-NEXT: paddd %xmm4, %xmm1
-; SSE2-NEXT: paddd %xmm4, %xmm3
+; SSE2-NEXT: paddd %xmm5, %xmm4
+; SSE2-NEXT: paddd %xmm1, %xmm2
+; SSE2-NEXT: paddd %xmm5, %xmm2
+; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: paddd %xmm1, %xmm3
; SSE2-NEXT: paddd %xmm5, %xmm3
-; SSE2-NEXT: paddd %xmm5, %xmm1
-; SSE2-NEXT: paddd %xmm3, %xmm1
-; SSE2-NEXT: paddd %xmm5, %xmm0
-; SSE2-NEXT: paddd %xmm2, %xmm5
-; SSE2-NEXT: paddd %xmm0, %xmm5
-; SSE2-NEXT: paddd %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; SSE2-NEXT: paddd %xmm1, %xmm0
; SSE2-NEXT: paddd %xmm5, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: retq
;
; AVX1-LABEL: sad_avx64i8_prefer_256:
@@ -585,22 +585,22 @@ define i32 @sad_avx64i8_prefer_256() nounwind "min-legal-vector-width"="256" "pr
; AVX1-NEXT: addq $64, %rax
; AVX1-NEXT: jne .LBB3_1
; AVX1-NEXT: # %bb.2: # %middle.block
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
; AVX1-NEXT: vpaddd %xmm4, %xmm4, %xmm5
-; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6
-; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm7
-; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm8
-; AVX1-NEXT: vpaddd %xmm0, %xmm8, %xmm8
-; AVX1-NEXT: vpaddd %xmm2, %xmm8, %xmm2
-; AVX1-NEXT: vpaddd %xmm7, %xmm0, %xmm0
+; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm0, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm6, %xmm2
-; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -613,26 +613,26 @@ define i32 @sad_avx64i8_prefer_256() nounwind "min-legal-vector-width"="256" "pr
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB3_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
; AVX2-NEXT: vmovdqa a+1024(%rax), %ymm3
; AVX2-NEXT: vpsadbw b+1024(%rax), %ymm3, %ymm3
-; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vmovdqa a+1056(%rax), %ymm3
; AVX2-NEXT: vpsadbw b+1056(%rax), %ymm3, %ymm3
-; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
; AVX2-NEXT: addq $64, %rax
; AVX2-NEXT: jne .LBB3_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm2
; AVX2-NEXT: vpaddd %ymm0, %ymm0, %ymm3
+; AVX2-NEXT: vpaddd %ymm3, %ymm2, %ymm2
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vpaddd %ymm3, %ymm0, %ymm0
-; AVX2-NEXT: vpaddd %ymm3, %ymm2, %ymm1
-; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -664,7 +664,7 @@ define i32 @sad_avx64i8_prefer_256() nounwind "min-legal-vector-width"="256" "pr
; AVX512F-NEXT: vpaddd %zmm0, %zmm0, %zmm0
; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -679,26 +679,26 @@ define i32 @sad_avx64i8_prefer_256() nounwind "min-legal-vector-width"="256" "pr
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB3_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
; AVX512BW-NEXT: vmovdqa a+1024(%rax), %ymm3
; AVX512BW-NEXT: vpsadbw b+1024(%rax), %ymm3, %ymm3
-; AVX512BW-NEXT: vpaddd %ymm1, %ymm3, %ymm1
+; AVX512BW-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; AVX512BW-NEXT: vmovdqa a+1056(%rax), %ymm3
; AVX512BW-NEXT: vpsadbw b+1056(%rax), %ymm3, %ymm3
-; AVX512BW-NEXT: vpaddd %ymm2, %ymm3, %ymm2
+; AVX512BW-NEXT: vpaddd %ymm1, %ymm3, %ymm1
; AVX512BW-NEXT: addq $64, %rax
; AVX512BW-NEXT: jne .LBB3_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vpaddd %ymm0, %ymm2, %ymm2
; AVX512BW-NEXT: vpaddd %ymm0, %ymm0, %ymm3
+; AVX512BW-NEXT: vpaddd %ymm3, %ymm2, %ymm2
; AVX512BW-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX512BW-NEXT: vpaddd %ymm3, %ymm0, %ymm0
-; AVX512BW-NEXT: vpaddd %ymm3, %ymm2, %ymm1
-; AVX512BW-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT: vpaddd %ymm0, %ymm2, %ymm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1162,10 +1162,12 @@ define dso_local i32 @sad_unroll_nonzero_initial(ptr %arg, ptr %arg1, ptr %arg2,
; AVX2-NEXT: vmovdqu (%rdx), %xmm1
; AVX2-NEXT: vpsadbw (%rcx), %xmm1, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [1,0,0,0]
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: incl %eax
; AVX2-NEXT: retq
;
; AVX512-LABEL: sad_unroll_nonzero_initial:
@@ -1176,11 +1178,9 @@ define dso_local i32 @sad_unroll_nonzero_initial(ptr %arg, ptr %arg1, ptr %arg2,
; AVX512-NEXT: vpsadbw (%rcx), %xmm1, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [1,0,0,0]
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: incl %eax
; AVX512-NEXT: retq
bb:
%tmp = load <16 x i8>, ptr %arg, align 1
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index 22c8cd692400c..5e3dffd931340 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -7,35 +7,37 @@ define <4 x i32> @test_compress_v4i32(<4 x i32> %vec, <4 x i1> %mask, <4 x i32>
; AVX2-LABEL: test_compress_v4i32:
; AVX2: # %bb.0:
; AVX2-NEXT: vpslld $31, %xmm1, %xmm1
-; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1
+; AVX2-NEXT: vpsrad $31, %xmm1, %xmm3
; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrd $1, %xmm1, %eax
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: movl %esi, %edi
-; AVX2-NEXT: subl %eax, %edi
-; AVX2-NEXT: vpextrd $2, %xmm1, %edx
-; AVX2-NEXT: subl %edx, %edi
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: subl %ecx, %edi
-; AVX2-NEXT: andl $3, %edi
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rsi, %rax
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rax, %rdx
+; AVX2-NEXT: vpsrld $31, %xmm1, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vmovd %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vmovd %xmm3, %edx
+; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: vpextrd $2, %xmm3, %esi
+; AVX2-NEXT: andl $1, %esi
+; AVX2-NEXT: addq %rcx, %rsi
+; AVX2-NEXT: vpextrd $3, %xmm3, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %rsi, %rdi
; AVX2-NEXT: vextractps $3, %xmm0, %r8d
-; AVX2-NEXT: cmpq $4, %rcx
-; AVX2-NEXT: cmovbl -24(%rsp,%rdi,4), %r8d
+; AVX2-NEXT: cmpq $4, %rdi
+; AVX2-NEXT: cmovbl -24(%rsp,%rax,4), %r8d
; AVX2-NEXT: vmovss %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rsi,4)
-; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rax,4)
-; AVX2-NEXT: andl $3, %edx
-; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rdx,4)
-; AVX2-NEXT: cmpq $3, %rcx
+; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rdx,4)
+; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx,4)
+; AVX2-NEXT: andl $3, %esi
+; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rsi,4)
+; AVX2-NEXT: cmpq $3, %rdi
; AVX2-NEXT: movl $3, %eax
-; AVX2-NEXT: cmovbq %rcx, %rax
+; AVX2-NEXT: cmovbq %rdi, %rax
; AVX2-NEXT: movl %r8d, -24(%rsp,%rax,4)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
@@ -68,40 +70,42 @@ define <4 x float> @test_compress_v4f32(<4 x float> %vec, <4 x i1> %mask, <4 x f
; AVX2-LABEL: test_compress_v4f32:
; AVX2: # %bb.0:
; AVX2-NEXT: vpslld $31, %xmm1, %xmm1
-; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1
+; AVX2-NEXT: vpsrad $31, %xmm1, %xmm3
; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrd $1, %xmm1, %edx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: movl %esi, %edi
-; AVX2-NEXT: subl %edx, %edi
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: subl %ecx, %edi
-; AVX2-NEXT: vpextrd $3, %xmm1, %eax
-; AVX2-NEXT: subl %eax, %edi
-; AVX2-NEXT: andl $3, %edi
+; AVX2-NEXT: vpsrld $31, %xmm1, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vmovd %xmm1, %eax
+; AVX2-NEXT: andl $3, %eax
; AVX2-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
; AVX2-NEXT: vmovss %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rsi,4)
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rsi, %rdx
-; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rdx,4)
+; AVX2-NEXT: vmovd %xmm3, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rax,4)
+; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx,4)
+; AVX2-NEXT: vpextrd $2, %xmm3, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $3, %eax
; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rcx,4)
-; AVX2-NEXT: cmpq $3, %rax
-; AVX2-NEXT: movl $3, %ecx
-; AVX2-NEXT: cmovbq %rax, %rcx
+; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rax,4)
+; AVX2-NEXT: cmpq $3, %rcx
+; AVX2-NEXT: movl $3, %eax
+; AVX2-NEXT: cmovbq %rcx, %rax
; AVX2-NEXT: ja .LBB1_2
; AVX2-NEXT: # %bb.1:
; AVX2-NEXT: vmovaps %xmm1, %xmm0
; AVX2-NEXT: .LBB1_2:
-; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rcx,4)
+; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rax,4)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
;
@@ -255,9 +259,9 @@ define <8 x i32> @test_compress_v8i32(<8 x i32> %vec, <8 x i1> %mask, <8 x i32>
; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %ecx
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
@@ -351,10 +355,10 @@ define <8 x float> @test_compress_v8f32(<8 x float> %vec, <8 x i1> %mask, <8 x f
; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vmovd %xmm2, %eax
+; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; AVX2-NEXT: vmovss %xmm0, (%rsp)
; AVX2-NEXT: vmovd %xmm3, %eax
@@ -448,9 +452,9 @@ define <4 x i64> @test_compress_v4i64(<4 x i64> %vec, <4 x i1> %mask, <4 x i64>
; AVX2-NEXT: vpsrlq $63, %ymm1, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: andl $3, %eax
; AVX2-NEXT: vpextrq $1, %xmm1, %rcx
; AVX2-NEXT: vmovq %xmm1, %rdx
@@ -520,10 +524,10 @@ define <4 x double> @test_compress_v4f64(<4 x double> %vec, <4 x i1> %mask, <4 x
; AVX2-NEXT: vpsrlq $63, %ymm3, %ymm1
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX2-NEXT: vpaddq %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vpextrq $1, %xmm1, %rax
-; AVX2-NEXT: vmovq %xmm1, %rcx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: andl $3, %eax
; AVX2-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
; AVX2-NEXT: vmovlpd %xmm0, (%rsp)
; AVX2-NEXT: vmovq %xmm3, %rax
@@ -600,15 +604,18 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpackusdw %ymm4, %ymm3, %ymm3
; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $1, %xmm3, %eax
-; AVX2-NEXT: vmovd %xmm3, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $15, %ecx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vmovd %xmm3, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $1, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
@@ -749,14 +756,17 @@ define <16 x float> @test_compress_v16f32(<16 x float> %vec, <16 x i1> %mask, <1
; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm4 = [255,255,255,255,255,255,255,255]
+; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpackusdw %ymm4, %ymm3, %ymm3
; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $1, %xmm3, %eax
-; AVX2-NEXT: vmovd %xmm3, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vmovd %xmm3, %eax
+; AVX2-NEXT: andl $15, %eax
; AVX2-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
; AVX2-NEXT: vmovss %xmm0, (%rsp)
; AVX2-NEXT: vmovd %xmm2, %eax
@@ -897,20 +907,27 @@ define <8 x i64> @test_compress_v8i64(<8 x i64> %vec, <8 x i1> %mask, <8 x i64>
; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
; AVX2-NEXT: vpaddq %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpackusdw %ymm4, %ymm3, %ymm3
+; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
+; AVX2-NEXT: vpackusdw %ymm4, %ymm3, %ymm3
; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm3, %rcx
-; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: andl $7, %eax
-; AVX2-NEXT: vpextrw $1, %xmm2, %ecx
-; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vmovd %xmm3, %ecx
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vpextrw $1, %xmm2, %eax
+; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm2, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: addq %rdx, %rax
; AVX2-NEXT: vpextrw $2, %xmm2, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rcx, %rsi
+; AVX2-NEXT: addq %rax, %rsi
; AVX2-NEXT: vpextrw $3, %xmm2, %edi
; AVX2-NEXT: andl $1, %edi
; AVX2-NEXT: addq %rsi, %rdi
@@ -929,11 +946,11 @@ define <8 x i64> @test_compress_v8i64(<8 x i64> %vec, <8 x i1> %mask, <8 x i64>
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX2-NEXT: vpextrq $1, %xmm2, %rbx
; AVX2-NEXT: cmpq $8, %r11
-; AVX2-NEXT: cmovbq (%rsp,%rax,8), %rbx
+; AVX2-NEXT: cmovbq (%rsp,%rcx,8), %rbx
; AVX2-NEXT: vmovq %xmm0, (%rsp)
; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rdx,8)
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, (%rsp,%rcx,8)
+; AVX2-NEXT: vmovq %xmm0, (%rsp,%rax,8)
; AVX2-NEXT: vpextrq $1, %xmm0, (%rsp,%rsi,8)
; AVX2-NEXT: andl $7, %edi
; AVX2-NEXT: vmovq %xmm1, (%rsp,%rdi,8)
@@ -989,12 +1006,19 @@ define <8 x double> @test_compress_v8f64(<8 x double> %vec, <8 x i1> %mask, <8 x
; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
; AVX2-NEXT: vpaddq %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm4 = [255,255,255,255]
+; AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpackusdw %ymm4, %ymm3, %ymm3
+; AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
+; AVX2-NEXT: vpackusdw %ymm4, %ymm3, %ymm3
; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm3, %rax
-; AVX2-NEXT: vmovq %xmm3, %rcx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vmovd %xmm3, %eax
+; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero
; AVX2-NEXT: vmovlps %xmm0, (%rsp)
; AVX2-NEXT: vmovd %xmm2, %eax
@@ -1076,139 +1100,109 @@ define <8 x double> @test_compress_v8f64(<8 x double> %vec, <8 x i1> %mask, <8 x
define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8> %passthru) nounwind {
; AVX2-LABEL: test_compress_v16i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
-; AVX2-NEXT: pushq %r15
-; AVX2-NEXT: pushq %r14
-; AVX2-NEXT: pushq %r13
-; AVX2-NEXT: pushq %r12
-; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: vpsllw $7, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: vpcmpgtb %xmm1, %xmm3, %xmm1
; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrb $1, %xmm1, %r13d
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: movl %esi, %eax
-; AVX2-NEXT: andb $1, %al
-; AVX2-NEXT: subb %r13b, %al
-; AVX2-NEXT: vpextrb $2, %xmm1, %edx
-; AVX2-NEXT: subb %dl, %al
-; AVX2-NEXT: vpextrb $3, %xmm1, %ebp
-; AVX2-NEXT: subb %bpl, %al
-; AVX2-NEXT: vpextrb $4, %xmm1, %r12d
-; AVX2-NEXT: subb %r12b, %al
-; AVX2-NEXT: vpextrb $5, %xmm1, %r15d
-; AVX2-NEXT: subb %r15b, %al
-; AVX2-NEXT: vpextrb $6, %xmm1, %r14d
-; AVX2-NEXT: subb %r14b, %al
-; AVX2-NEXT: vpextrb $7, %xmm1, %ebx
-; AVX2-NEXT: subb %bl, %al
-; AVX2-NEXT: vpextrb $8, %xmm1, %r11d
-; AVX2-NEXT: subb %r11b, %al
-; AVX2-NEXT: vpextrb $9, %xmm1, %r10d
-; AVX2-NEXT: subb %r10b, %al
-; AVX2-NEXT: vpextrb $10, %xmm1, %r9d
-; AVX2-NEXT: subb %r9b, %al
-; AVX2-NEXT: vpextrb $11, %xmm1, %r8d
-; AVX2-NEXT: subb %r8b, %al
-; AVX2-NEXT: vpextrb $12, %xmm1, %edi
-; AVX2-NEXT: subb %dil, %al
-; AVX2-NEXT: vpextrb $13, %xmm1, %ecx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: subb %cl, %al
-; AVX2-NEXT: vpextrb $14, %xmm1, %ecx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: subb %cl, %al
-; AVX2-NEXT: vpextrb $15, %xmm1, %ecx
-; AVX2-NEXT: subb %cl, %al
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vmovd %xmm2, %eax
; AVX2-NEXT: andl $15, %eax
-; AVX2-NEXT: movzbl -40(%rsp,%rax), %eax
-; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX2-NEXT: movzbl -24(%rsp,%rax), %eax
; AVX2-NEXT: vpextrb $0, %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: vpextrb $1, %xmm0, -40(%rsp,%rsi)
-; AVX2-NEXT: andl $1, %r13d
-; AVX2-NEXT: addq %rsi, %r13
-; AVX2-NEXT: vpextrb $2, %xmm0, -40(%rsp,%r13)
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vpextrb $1, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $1, %xmm1, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %r13, %rdx
-; AVX2-NEXT: vpextrb $3, %xmm0, -40(%rsp,%rdx)
-; AVX2-NEXT: andl $1, %ebp
-; AVX2-NEXT: addq %rdx, %rbp
-; AVX2-NEXT: vpextrb $4, %xmm0, -40(%rsp,%rbp)
-; AVX2-NEXT: andl $1, %r12d
-; AVX2-NEXT: addq %rbp, %r12
-; AVX2-NEXT: andl $1, %r15d
-; AVX2-NEXT: addq %r12, %r15
-; AVX2-NEXT: # kill: def $r12d killed $r12d killed $r12 def $r12
-; AVX2-NEXT: andl $15, %r12d
-; AVX2-NEXT: vpextrb $5, %xmm0, -40(%rsp,%r12)
-; AVX2-NEXT: andl $1, %r14d
-; AVX2-NEXT: addq %r15, %r14
-; AVX2-NEXT: # kill: def $r15d killed $r15d killed $r15 def $r15
-; AVX2-NEXT: andl $15, %r15d
-; AVX2-NEXT: vpextrb $6, %xmm0, -40(%rsp,%r15)
-; AVX2-NEXT: andl $1, %ebx
-; AVX2-NEXT: addq %r14, %rbx
-; AVX2-NEXT: # kill: def $r14d killed $r14d killed $r14 def $r14
-; AVX2-NEXT: andl $15, %r14d
-; AVX2-NEXT: vpextrb $7, %xmm0, -40(%rsp,%r14)
-; AVX2-NEXT: andl $1, %r11d
-; AVX2-NEXT: addq %rbx, %r11
-; AVX2-NEXT: # kill: def $ebx killed $ebx killed $rbx def $rbx
-; AVX2-NEXT: andl $15, %ebx
-; AVX2-NEXT: vpextrb $8, %xmm0, -40(%rsp,%rbx)
-; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r11, %r10
-; AVX2-NEXT: # kill: def $r11d killed $r11d killed $r11 def $r11
-; AVX2-NEXT: andl $15, %r11d
-; AVX2-NEXT: vpextrb $9, %xmm0, -40(%rsp,%r11)
-; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r10, %r9
-; AVX2-NEXT: # kill: def $r10d killed $r10d killed $r10 def $r10
-; AVX2-NEXT: andl $15, %r10d
-; AVX2-NEXT: vpextrb $10, %xmm0, -40(%rsp,%r10)
-; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %r9, %r8
-; AVX2-NEXT: # kill: def $r9d killed $r9d killed $r9 def $r9
-; AVX2-NEXT: andl $15, %r9d
-; AVX2-NEXT: vpextrb $11, %xmm0, -40(%rsp,%r9)
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %r8, %rdi
-; AVX2-NEXT: # kill: def $r8d killed $r8d killed $r8 def $r8
-; AVX2-NEXT: andl $15, %r8d
-; AVX2-NEXT: vpextrb $12, %xmm0, -40(%rsp,%r8)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rdi, %rsi
-; AVX2-NEXT: # kill: def $edi killed $edi killed $rdi def $rdi
-; AVX2-NEXT: andl $15, %edi
-; AVX2-NEXT: vpextrb $13, %xmm0, -40(%rsp,%rdi)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rsi, %rax
-; AVX2-NEXT: # kill: def $esi killed $esi killed $rsi def $rsi
-; AVX2-NEXT: andl $15, %esi
-; AVX2-NEXT: vpextrb $14, %xmm0, -40(%rsp,%rsi)
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: vpextrb $2, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $2, %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
-; AVX2-NEXT: andl $15, %eax
-; AVX2-NEXT: vpextrb $15, %xmm0, -40(%rsp,%rax)
-; AVX2-NEXT: cmpq $15, %rcx
-; AVX2-NEXT: movl $15, %eax
-; AVX2-NEXT: cmovbq %rcx, %rax
-; AVX2-NEXT: vpextrb $15, %xmm0, %ecx
-; AVX2-NEXT: cmovbel {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Folded Reload
-; AVX2-NEXT: movb %cl, -40(%rsp,%rax)
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: vpextrb $3, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $3, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: vpextrb $4, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $4, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: vpextrb $5, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $5, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $6, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
+; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vpextrb $6, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $7, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $7, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $8, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
+; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vpextrb $8, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $9, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $9, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $10, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
+; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vpextrb $10, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $11, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $11, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $12, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
+; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vpextrb $12, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $13, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $13, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $14, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
+; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vpextrb $14, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $15, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $15, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: cmpq $15, %rdx
+; AVX2-NEXT: movl $15, %ecx
+; AVX2-NEXT: cmovbq %rdx, %rcx
+; AVX2-NEXT: vpextrb $15, %xmm0, %edx
+; AVX2-NEXT: cmovbel %eax, %edx
+; AVX2-NEXT: movb %dl, -24(%rsp,%rcx)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
-; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: popq %r12
-; AVX2-NEXT: popq %r13
-; AVX2-NEXT: popq %r14
-; AVX2-NEXT: popq %r15
-; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_compress_v16i8:
@@ -1243,55 +1237,63 @@ define <8 x i16> @test_compress_v8i16(<8 x i16> %vec, <8 x i1> %mask, <8 x i16>
; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vpextrw $1, %xmm1, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: vmovd %xmm1, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: leal (%rcx,%rax), %esi
-; AVX2-NEXT: vpextrw $2, %xmm1, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: vpextrw $3, %xmm1, %edx
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: leal (%rdi,%rdx), %r10d
-; AVX2-NEXT: addl %esi, %r10d
-; AVX2-NEXT: vpextrw $4, %xmm1, %r9d
-; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: vpextrw $5, %xmm1, %esi
+; AVX2-NEXT: vmovd %xmm1, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: leal (%r9,%rsi), %r11d
-; AVX2-NEXT: vpextrw $6, %xmm1, %r8d
-; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addl %r8d, %r11d
-; AVX2-NEXT: addl %r10d, %r11d
-; AVX2-NEXT: vpextrw $7, %xmm1, %r10d
+; AVX2-NEXT: vmovd %esi, %xmm2
+; AVX2-NEXT: vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $2, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addl %r10d, %r11d
-; AVX2-NEXT: andl $7, %r11d
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: addq %rax, %rdi
+; AVX2-NEXT: vpinsrw $2, %r10d, %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $3, %xmm1, %r9d
+; AVX2-NEXT: andl $1, %r9d
+; AVX2-NEXT: vpinsrw $3, %r9d, %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $4, %xmm1, %r8d
+; AVX2-NEXT: andl $1, %r8d
+; AVX2-NEXT: vpinsrw $4, %r8d, %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $5, %xmm1, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: vpinsrw $5, %edi, %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $6, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: vpinsrw $6, %edx, %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $7, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vpinsrw $7, %ecx, %xmm2, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpaddw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX2-NEXT: vpaddw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpsrld $16, %xmm1, %xmm2
+; AVX2-NEXT: vpaddw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vmovd %xmm1, %ebx
+; AVX2-NEXT: andl $7, %ebx
+; AVX2-NEXT: addq %rsi, %rax
+; AVX2-NEXT: addq %rax, %r10
+; AVX2-NEXT: addq %r10, %r9
+; AVX2-NEXT: addq %r9, %r8
+; AVX2-NEXT: addq %r8, %rdi
; AVX2-NEXT: addq %rdi, %rdx
-; AVX2-NEXT: addq %rdx, %r9
-; AVX2-NEXT: addq %r9, %rsi
-; AVX2-NEXT: addq %rsi, %r8
-; AVX2-NEXT: addq %r8, %r10
-; AVX2-NEXT: vpextrw $7, %xmm0, %ebx
-; AVX2-NEXT: cmpq $8, %r10
-; AVX2-NEXT: cmovbw -16(%rsp,%r11,2), %bx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: vpextrw $7, %xmm0, %r11d
+; AVX2-NEXT: cmpq $8, %rcx
+; AVX2-NEXT: cmovbw -16(%rsp,%rbx,2), %r11w
; AVX2-NEXT: vpextrw $0, %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrw $1, %xmm0, -16(%rsp,%rcx,2)
+; AVX2-NEXT: vpextrw $1, %xmm0, -16(%rsp,%rsi,2)
; AVX2-NEXT: vpextrw $2, %xmm0, -16(%rsp,%rax,2)
-; AVX2-NEXT: vpextrw $3, %xmm0, -16(%rsp,%rdi,2)
-; AVX2-NEXT: andl $7, %edx
-; AVX2-NEXT: vpextrw $4, %xmm0, -16(%rsp,%rdx,2)
+; AVX2-NEXT: vpextrw $3, %xmm0, -16(%rsp,%r10,2)
; AVX2-NEXT: andl $7, %r9d
-; AVX2-NEXT: vpextrw $5, %xmm0, -16(%rsp,%r9,2)
-; AVX2-NEXT: andl $7, %esi
-; AVX2-NEXT: vpextrw $6, %xmm0, -16(%rsp,%rsi,2)
+; AVX2-NEXT: vpextrw $4, %xmm0, -16(%rsp,%r9,2)
; AVX2-NEXT: andl $7, %r8d
-; AVX2-NEXT: vpextrw $7, %xmm0, -16(%rsp,%r8,2)
-; AVX2-NEXT: cmpq $7, %r10
+; AVX2-NEXT: vpextrw $5, %xmm0, -16(%rsp,%r8,2)
+; AVX2-NEXT: andl $7, %edi
+; AVX2-NEXT: vpextrw $6, %xmm0, -16(%rsp,%rdi,2)
+; AVX2-NEXT: andl $7, %edx
+; AVX2-NEXT: vpextrw $7, %xmm0, -16(%rsp,%rdx,2)
+; AVX2-NEXT: cmpq $7, %rcx
; AVX2-NEXT: movl $7, %eax
-; AVX2-NEXT: cmovbq %r10, %rax
+; AVX2-NEXT: cmovbq %rcx, %rax
; AVX2-NEXT: movl %eax, %eax
-; AVX2-NEXT: movw %bx, -16(%rsp,%rax,2)
+; AVX2-NEXT: movw %r11w, -16(%rsp,%rax,2)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: retq
@@ -1335,16 +1337,11 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm4
; AVX2-NEXT: vpand %xmm2, %xmm3, %xmm2
; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm2[8,9,10,11,12,13,14,15,12,13],zero,zero,xmm2[14,15],zero,zero
-; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
-; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpsrld $16, %xmm2, %xmm4
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpextrb $1, %xmm2, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: addb %al, %cl
-; AVX2-NEXT: movzbl %cl, %eax
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpsadbw %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vmovd %xmm2, %eax
; AVX2-NEXT: andl $31, %eax
; AVX2-NEXT: movzbl (%rsp,%rax), %eax
; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp)
@@ -1527,13 +1524,12 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $31, %ecx
; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: vpextrb $15, %xmm0, %ecx
-; AVX2-NEXT: cmpq $32, %rdx
-; AVX2-NEXT: cmovbl %eax, %ecx
; AVX2-NEXT: cmpq $31, %rdx
-; AVX2-NEXT: movl $31, %eax
-; AVX2-NEXT: cmovbq %rdx, %rax
-; AVX2-NEXT: movb %cl, (%rsp,%rax)
+; AVX2-NEXT: movl $31, %ecx
+; AVX2-NEXT: cmovbq %rdx, %rcx
+; AVX2-NEXT: vpextrb $15, %xmm0, %edx
+; AVX2-NEXT: cmovbel %eax, %edx
+; AVX2-NEXT: movb %dl, (%rsp,%rcx)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: movq %rbp, %rsp
; AVX2-NEXT: popq %rbp
@@ -1556,21 +1552,16 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k2} {z}
; AVX512F-NEXT: vpmovdb %zmm3, (%rsp)
; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
-; AVX512F-NEXT: vextracti64x4 $1, %zmm3, %ymm4
-; AVX512F-NEXT: vpaddd %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX512F-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX512F-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512F-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
; AVX512F-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT: vpextrd $1, %xmm3, %eax
-; AVX512F-NEXT: vmovd %xmm3, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vpaddq %xmm4, %xmm3, %xmm3
+; AVX512F-NEXT: vmovq %xmm3, %rax
+; AVX512F-NEXT: andl $31, %eax
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vpmovdb %zmm0, (%rsp,%rcx)
+; AVX512F-NEXT: vpmovdb %zmm0, (%rsp,%rax)
; AVX512F-NEXT: vpsllw $7, %ymm1, %ymm0
; AVX512F-NEXT: vpblendvb %ymm0, (%rsp), %ymm2, %ymm0
; AVX512F-NEXT: movq %rbp, %rsp
@@ -1611,11 +1602,11 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpaddw %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[1,1,1,1]
; AVX2-NEXT: vpaddw %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $1, %xmm2, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $15, %ecx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpsrld $16, %xmm2, %xmm3
+; AVX2-NEXT: vpaddw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vmovd %xmm2, %eax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrw $1, %xmm1, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm1, %ecx
@@ -1755,15 +1746,15 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: andq $-32, %rsp
; AVX2-NEXT: subq $96, %rsp
-; AVX2-NEXT: movl %r9d, %r10d
-; AVX2-NEXT: movl %r8d, %r9d
-; AVX2-NEXT: movl %ecx, %r8d
+; AVX2-NEXT: # kill: def $r9d killed $r9d def $r9
+; AVX2-NEXT: # kill: def $r8d killed $r8d def $r8
+; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
; AVX2-NEXT: # kill: def $esi killed $esi def $rsi
; AVX2-NEXT: # kill: def $edi killed $edi def $rdi
; AVX2-NEXT: movzbl 360(%rbp), %eax
-; AVX2-NEXT: movzbl 352(%rbp), %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm4
+; AVX2-NEXT: movzbl 352(%rbp), %r10d
+; AVX2-NEXT: vmovd %r10d, %xmm4
; AVX2-NEXT: vpinsrb $1, %eax, %xmm4, %xmm4
; AVX2-NEXT: movzbl 368(%rbp), %eax
; AVX2-NEXT: vpinsrb $2, %eax, %xmm4, %xmm4
@@ -1829,83 +1820,77 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: vmovd %edi, %xmm5
; AVX2-NEXT: vpinsrb $1, %esi, %xmm5, %xmm5
; AVX2-NEXT: vpinsrb $2, %edx, %xmm5, %xmm5
-; AVX2-NEXT: vpinsrb $3, %r8d, %xmm5, %xmm5
-; AVX2-NEXT: vpinsrb $4, %r9d, %xmm5, %xmm5
-; AVX2-NEXT: vpinsrb $5, %r10d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 16(%rbp), %r11d
-; AVX2-NEXT: vpinsrb $6, %r11d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 24(%rbp), %ebx
-; AVX2-NEXT: vpinsrb $7, %ebx, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 32(%rbp), %r14d
-; AVX2-NEXT: vpinsrb $8, %r14d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 40(%rbp), %r15d
-; AVX2-NEXT: vpinsrb $9, %r15d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 48(%rbp), %r12d
-; AVX2-NEXT: vpinsrb $10, %r12d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 56(%rbp), %r13d
-; AVX2-NEXT: vpinsrb $11, %r13d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 64(%rbp), %eax
-; AVX2-NEXT: vpinsrb $12, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 72(%rbp), %eax
-; AVX2-NEXT: vpinsrb $13, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 80(%rbp), %eax
-; AVX2-NEXT: vpinsrb $14, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 88(%rbp), %eax
-; AVX2-NEXT: vpinsrb $15, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 96(%rbp), %eax
-; AVX2-NEXT: vmovd %eax, %xmm6
-; AVX2-NEXT: movzbl 104(%rbp), %eax
-; AVX2-NEXT: vpinsrb $1, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 112(%rbp), %eax
-; AVX2-NEXT: vpinsrb $2, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 120(%rbp), %eax
-; AVX2-NEXT: vpinsrb $3, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 128(%rbp), %eax
-; AVX2-NEXT: vpinsrb $4, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 136(%rbp), %eax
-; AVX2-NEXT: vpinsrb $5, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 144(%rbp), %eax
-; AVX2-NEXT: vpinsrb $6, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 152(%rbp), %eax
-; AVX2-NEXT: vpinsrb $7, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 160(%rbp), %eax
-; AVX2-NEXT: vpinsrb $8, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 168(%rbp), %eax
-; AVX2-NEXT: vpinsrb $9, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 176(%rbp), %eax
-; AVX2-NEXT: vpinsrb $10, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 184(%rbp), %eax
-; AVX2-NEXT: vpinsrb $11, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 192(%rbp), %eax
-; AVX2-NEXT: vpinsrb $12, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 200(%rbp), %eax
-; AVX2-NEXT: vpinsrb $13, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 208(%rbp), %eax
-; AVX2-NEXT: vpinsrb $14, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 216(%rbp), %eax
-; AVX2-NEXT: vpinsrb $15, %eax, %xmm6, %xmm6
+; AVX2-NEXT: vpinsrb $3, %ecx, %xmm5, %xmm5
+; AVX2-NEXT: vpinsrb $4, %r8d, %xmm5, %xmm5
+; AVX2-NEXT: vpinsrb $5, %r9d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 16(%rbp), %eax
+; AVX2-NEXT: vpinsrb $6, %eax, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 24(%rbp), %r10d
+; AVX2-NEXT: vpinsrb $7, %r10d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 32(%rbp), %r11d
+; AVX2-NEXT: vpinsrb $8, %r11d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 40(%rbp), %ebx
+; AVX2-NEXT: vpinsrb $9, %ebx, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 48(%rbp), %r14d
+; AVX2-NEXT: vpinsrb $10, %r14d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 56(%rbp), %r15d
+; AVX2-NEXT: vpinsrb $11, %r15d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 64(%rbp), %r12d
+; AVX2-NEXT: vpinsrb $12, %r12d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 72(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $13, %r13d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 80(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $14, %r13d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 88(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $15, %r13d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 96(%rbp), %r13d
+; AVX2-NEXT: vmovd %r13d, %xmm6
+; AVX2-NEXT: movzbl 104(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $1, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 112(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $2, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 120(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $3, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 128(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $4, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 136(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $5, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 144(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $6, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 152(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $7, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 160(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $8, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 168(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $9, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 176(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $10, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 184(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $11, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 192(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $12, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 200(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $13, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 208(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $14, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 216(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $15, %r13d, %xmm6, %xmm6
; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm5, %ymm5
-; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX2-NEXT: vpand %ymm6, %ymm5, %ymm5
-; AVX2-NEXT: vpand %ymm6, %ymm4, %ymm4
-; AVX2-NEXT: vpaddb %ymm4, %ymm5, %ymm4
-; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm4[8,9,10,11,12,13,14,15,12,13],zero,zero,xmm4[14,15],zero,zero
-; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
-; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpsrld $16, %xmm4, %xmm5
-; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpextrb $1, %xmm4, %ecx
-; AVX2-NEXT: vmovd %xmm4, %eax
-; AVX2-NEXT: addb %cl, %al
; AVX2-NEXT: vmovaps %ymm3, {{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps %ymm2, (%rsp)
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: movzbl (%rsp,%rax), %eax
-; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX2-NEXT: vpand %ymm2, %ymm5, %ymm3
+; AVX2-NEXT: vpand %ymm2, %ymm4, %ymm2
+; AVX2-NEXT: vpaddb %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT: vpaddb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpaddb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vmovd %xmm2, %r13d
+; AVX2-NEXT: andl $63, %r13d
+; AVX2-NEXT: movzbl (%rsp,%r13), %r13d
; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp)
; AVX2-NEXT: andl $1, %edi
; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rdi)
@@ -1915,53 +1900,52 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: addq %rsi, %rdx
; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%rdx)
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%rcx)
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rdx, %r8
-; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%r8)
+; AVX2-NEXT: addq %rcx, %r8
+; AVX2-NEXT: movl %r8d, %ecx
+; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rcx)
; AVX2-NEXT: andl $1, %r9d
; AVX2-NEXT: addq %r8, %r9
-; AVX2-NEXT: movl %r9d, %eax
-; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
-; AVX2-NEXT: movzbl %r11b, %eax
+; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %r10, %rax
-; AVX2-NEXT: # kill: def $r10d killed $r10d killed $r10 def $r10
-; AVX2-NEXT: andl $63, %r10d
-; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%r10)
-; AVX2-NEXT: movzbl %bl, %ecx
+; AVX2-NEXT: addq %r9, %rax
+; AVX2-NEXT: # kill: def $r9d killed $r9d killed $r9 def $r9
+; AVX2-NEXT: andl $63, %r9d
+; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%r9)
+; AVX2-NEXT: movzbl %r10b, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl %r14b, %eax
+; AVX2-NEXT: movzbl %r11b, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl %r15b, %ecx
+; AVX2-NEXT: movzbl %bl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl %r12b, %eax
+; AVX2-NEXT: movzbl %r14b, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl %r13b, %ecx
+; AVX2-NEXT: movzbl %r15b, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl 64(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movzbl %r12b, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
@@ -2328,7 +2312,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rax)
; AVX2-NEXT: vpextrb $15, %xmm0, %eax
; AVX2-NEXT: cmpq $64, %rcx
-; AVX2-NEXT: cmovbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; AVX2-NEXT: cmovbl %r13d, %eax
; AVX2-NEXT: cmpq $63, %rcx
; AVX2-NEXT: movl $63, %edx
; AVX2-NEXT: cmovbq %rcx, %rdx
@@ -2366,130 +2350,119 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: vpinsrb $13, 456(%rbp), %xmm2, %xmm2
; AVX512F-NEXT: vpinsrb $14, 464(%rbp), %xmm2, %xmm2
; AVX512F-NEXT: vpinsrb $15, 472(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm3 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT: vptestmd %zmm3, %zmm2, %k1
-; AVX512F-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512F-NEXT: vpinsrb $1, 232(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $2, 240(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $3, 248(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $4, 256(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $5, 264(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $6, 272(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $7, 280(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $8, 288(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $9, 296(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $10, 304(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $11, 312(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $12, 320(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $13, 328(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $14, 336(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $15, 344(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512F-NEXT: vptestmd %zmm3, %zmm2, %k3
-; AVX512F-NEXT: vmovd {{.*#+}} xmm2 = mem[0],zero,zero,zero
-; AVX512F-NEXT: vpinsrb $1, 104(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $2, 112(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $3, 120(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $4, 128(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $5, 136(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $6, 144(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $7, 152(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $8, 160(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $9, 168(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $10, 176(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $11, 184(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $12, 192(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $13, 200(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $14, 208(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $15, 216(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512F-NEXT: vptestmd %zmm3, %zmm2, %k2
-; AVX512F-NEXT: vmovd %edi, %xmm2
-; AVX512F-NEXT: vpinsrb $1, %esi, %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $2, %edx, %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $3, %ecx, %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $4, %r8d, %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $5, %r9d, %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $6, 16(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $7, 24(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $8, 32(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $9, 40(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $10, 48(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $11, 56(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $12, 64(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $13, 72(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $14, 80(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpinsrb $15, 88(%rbp), %xmm2, %xmm2
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
-; AVX512F-NEXT: vptestmd %zmm3, %zmm2, %k4
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT: vpcompressd %zmm2, %zmm2 {%k4} {z}
-; AVX512F-NEXT: vpmovdb %zmm2, (%rsp)
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k4} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm2, %zmm3
-; AVX512F-NEXT: vextracti64x4 $1, %zmm3, %ymm4
-; AVX512F-NEXT: vpaddd %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX512F-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT: vpextrd $1, %xmm3, %eax
-; AVX512F-NEXT: vmovd %xmm3, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $31, %ecx
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm3
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero,xmm2[8],zero,zero,zero,xmm2[9],zero,zero,zero,xmm2[10],zero,zero,zero,xmm2[11],zero,zero,zero,xmm2[12],zero,zero,zero,xmm2[13],zero,zero,zero,xmm2[14],zero,zero,zero,xmm2[15],zero,zero,zero
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT: vptestmd %zmm2, %zmm3, %k1
+; AVX512F-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vpinsrb $1, 232(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $2, 240(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $3, 248(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $4, 256(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $5, 264(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $6, 272(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $7, 280(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $8, 288(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $9, 296(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $10, 304(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $11, 312(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $12, 320(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $13, 328(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $14, 336(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $15, 344(%rbp), %xmm3, %xmm3
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
-; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k2} {z}
-; AVX512F-NEXT: vpmovdb %zmm3, (%rsp,%rcx)
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; AVX512F-NEXT: vptestmd %zmm2, %zmm3, %k2
+; AVX512F-NEXT: vmovd {{.*#+}} xmm3 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vpinsrb $1, 104(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $2, 112(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $3, 120(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $4, 128(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $5, 136(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $6, 144(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $7, 152(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $8, 160(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $9, 168(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $10, 176(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $11, 184(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $12, 192(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $13, 200(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $14, 208(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $15, 216(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
+; AVX512F-NEXT: vptestmd %zmm2, %zmm3, %k3
+; AVX512F-NEXT: vmovd %edi, %xmm3
+; AVX512F-NEXT: vpinsrb $1, %esi, %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $2, %edx, %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $3, %ecx, %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $4, %r8d, %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $5, %r9d, %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $6, 16(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $7, 24(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $8, 32(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $9, 40(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $10, 48(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $11, 56(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $12, 64(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $13, 72(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $14, 80(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpinsrb $15, 88(%rbp), %xmm3, %xmm3
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
+; AVX512F-NEXT: vptestmd %zmm2, %zmm3, %k4
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
-; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k3} {z}
-; AVX512F-NEXT: vpmovdb %zmm3, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k3} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm3, %zmm4
-; AVX512F-NEXT: vextracti64x4 $1, %zmm4, %ymm5
-; AVX512F-NEXT: vpaddd %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k4} {z}
+; AVX512F-NEXT: vpmovdb %zmm3, (%rsp)
+; AVX512F-NEXT: vmovdqa32 %zmm2, %zmm3 {%k4} {z}
+; AVX512F-NEXT: vpmovdb %zmm3, %xmm4
+; AVX512F-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512F-NEXT: vpsadbw %xmm3, %xmm4, %xmm4
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
+; AVX512F-NEXT: vpaddq %xmm5, %xmm4, %xmm4
+; AVX512F-NEXT: vmovq %xmm4, %rax
+; AVX512F-NEXT: andl $31, %eax
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm4
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512F-NEXT: vpcompressd %zmm4, %zmm4 {%k3} {z}
+; AVX512F-NEXT: vpmovdb %zmm4, (%rsp,%rax)
+; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512F-NEXT: vpcompressd %zmm4, %zmm4 {%k2} {z}
+; AVX512F-NEXT: vpmovdb %zmm4, {{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vmovdqa32 %zmm2, %zmm4 {%k2} {z}
+; AVX512F-NEXT: vpmovdb %zmm4, %xmm4
+; AVX512F-NEXT: vpsadbw %xmm3, %xmm4, %xmm4
; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
-; AVX512F-NEXT: vmovd %xmm4, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vpaddq %xmm5, %xmm4, %xmm4
+; AVX512F-NEXT: vmovq %xmm4, %rax
+; AVX512F-NEXT: andl $31, %eax
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vpmovdb %zmm0, 32(%rsp,%rcx)
-; AVX512F-NEXT: vmovdqa (%rsp), %ymm0
-; AVX512F-NEXT: vmovdqa %ymm0, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm0, %zmm4
-; AVX512F-NEXT: vpsubd %zmm2, %zmm4, %zmm4
-; AVX512F-NEXT: vextracti64x4 $1, %zmm4, %ymm5
-; AVX512F-NEXT: vpaddd %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
-; AVX512F-NEXT: vmovd %xmm4, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $63, %ecx
-; AVX512F-NEXT: vmovdqa {{[0-9]+}}(%rsp), %ymm4
-; AVX512F-NEXT: vmovdqa %ymm4, 64(%rsp,%rcx)
-; AVX512F-NEXT: vpmovdb %zmm3, %xmm3
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 {%k1} {z} = -1
-; AVX512F-NEXT: vpmovdb %zmm4, %xmm4
-; AVX512F-NEXT: vinserti128 $1, %xmm4, %ymm3, %ymm3
-; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm4
-; AVX512F-NEXT: vpblendvb %ymm3, {{[0-9]+}}(%rsp), %ymm4, %ymm3
+; AVX512F-NEXT: vpmovdb %zmm0, 32(%rsp,%rax)
+; AVX512F-NEXT: vmovaps (%rsp), %ymm0
+; AVX512F-NEXT: vmovaps %ymm0, {{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vmovdqa32 %zmm2, %zmm0 {%k3} {z}
+; AVX512F-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k4} {z} = -1
; AVX512F-NEXT: vpmovdb %zmm2, %xmm2
+; AVX512F-NEXT: vpsubb %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX512F-NEXT: vpaddq %xmm3, %xmm0, %xmm0
+; AVX512F-NEXT: vmovq %xmm0, %rax
+; AVX512F-NEXT: andl $63, %eax
+; AVX512F-NEXT: vmovdqa {{[0-9]+}}(%rsp), %ymm0
+; AVX512F-NEXT: vmovdqa %ymm0, 64(%rsp,%rax)
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1
; AVX512F-NEXT: vpmovdb %zmm0, %xmm0
-; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm2, %ymm0
-; AVX512F-NEXT: vpblendvb %ymm0, {{[0-9]+}}(%rsp), %ymm1, %ymm0
-; AVX512F-NEXT: vinserti64x4 $1, %ymm3, %zmm0, %zmm0
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k1} {z} = -1
+; AVX512F-NEXT: vpmovdb %zmm3, %xmm3
+; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm0, %ymm0
+; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm3
+; AVX512F-NEXT: vpblendvb %ymm0, {{[0-9]+}}(%rsp), %ymm3, %ymm0
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k3} {z} = -1
+; AVX512F-NEXT: vpmovdb %zmm3, %xmm3
+; AVX512F-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX512F-NEXT: vpblendvb %ymm2, {{[0-9]+}}(%rsp), %ymm1, %ymm1
+; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
; AVX512F-NEXT: movq %rbp, %rsp
; AVX512F-NEXT: popq %rbp
; AVX512F-NEXT: retq
@@ -2527,16 +2500,14 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX2-NEXT: vpand %ymm5, %ymm6, %ymm5
; AVX2-NEXT: vpaddw %ymm4, %ymm5, %ymm4
; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX2-NEXT: vpaddw %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vpackuswb %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX2-NEXT: vpsadbw %xmm5, %xmm4, %xmm4
; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; AVX2-NEXT: vpaddw %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
-; AVX2-NEXT: vpaddw %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpextrw $1, %xmm4, %eax
-; AVX2-NEXT: vmovd %xmm4, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $31, %ecx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpaddq %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vmovq %xmm4, %rax
+; AVX2-NEXT: andl $31, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $1, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
@@ -2761,35 +2732,30 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX512F-NEXT: andq $-64, %rsp
; AVX512F-NEXT: subq $128, %rsp
; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
-; AVX512F-NEXT: vpmovsxbd %xmm5, %zmm5
-; AVX512F-NEXT: vpslld $31, %zmm5, %zmm5
-; AVX512F-NEXT: vptestmd %zmm5, %zmm5, %k1
-; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm5
-; AVX512F-NEXT: vpslld $31, %zmm5, %zmm5
-; AVX512F-NEXT: vptestmd %zmm5, %zmm5, %k2
-; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm5 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT: vpcompressd %zmm5, %zmm5 {%k2} {z}
-; AVX512F-NEXT: vpmovdw %zmm5, (%rsp)
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512F-NEXT: vpmovsxbd %xmm4, %zmm4
+; AVX512F-NEXT: vpslld $31, %zmm4, %zmm4
+; AVX512F-NEXT: vptestmd %zmm4, %zmm4, %k1
+; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm4
+; AVX512F-NEXT: vpslld $31, %zmm4, %zmm4
+; AVX512F-NEXT: vptestmd %zmm4, %zmm4, %k2
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT: vpcompressd %zmm4, %zmm4 {%k2} {z}
+; AVX512F-NEXT: vpmovdw %zmm4, (%rsp)
; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm5
-; AVX512F-NEXT: vpaddd %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX512F-NEXT: vpaddd %xmm5, %xmm1, %xmm1
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm5, %xmm1, %xmm1
-; AVX512F-NEXT: vpextrd $1, %xmm1, %eax
-; AVX512F-NEXT: vmovd %xmm1, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512F-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; AVX512F-NEXT: vpaddq %xmm4, %xmm1, %xmm1
+; AVX512F-NEXT: vmovq %xmm1, %rax
+; AVX512F-NEXT: andl $31, %eax
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vpmovdw %zmm0, (%rsp,%rcx,2)
+; AVX512F-NEXT: vpmovdw %zmm0, (%rsp,%rax,2)
; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm0
-; AVX512F-NEXT: vpsllw $15, %ymm4, %ymm1
+; AVX512F-NEXT: vpsllw $15, %ymm5, %ymm1
; AVX512F-NEXT: vpsraw $15, %ymm1, %ymm1
; AVX512F-NEXT: vpblendvb %ymm1, {{[0-9]+}}(%rsp), %ymm0, %ymm0
; AVX512F-NEXT: vpsllw $15, %ymm3, %ymm1
@@ -3293,116 +3259,104 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512F-NEXT: vpinsrb $13, 456(%rbp), %xmm4, %xmm4
; AVX512F-NEXT: vpinsrb $14, 464(%rbp), %xmm4, %xmm4
; AVX512F-NEXT: vpinsrb $15, 472(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
-; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm5 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX512F-NEXT: vptestmd %zmm5, %zmm4, %k1
-; AVX512F-NEXT: vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; AVX512F-NEXT: vpinsrb $1, 232(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $2, 240(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $3, 248(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $4, 256(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $5, 264(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $6, 272(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $7, 280(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $8, 288(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $9, 296(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $10, 304(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $11, 312(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $12, 320(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $13, 328(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $14, 336(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $15, 344(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
-; AVX512F-NEXT: vptestmd %zmm5, %zmm4, %k3
-; AVX512F-NEXT: vmovd {{.*#+}} xmm4 = mem[0],zero,zero,zero
-; AVX512F-NEXT: vpinsrb $1, 104(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $2, 112(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $3, 120(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $4, 128(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $5, 136(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $6, 144(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $7, 152(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $8, 160(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $9, 168(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $10, 176(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $11, 184(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $12, 192(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $13, 200(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $14, 208(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $15, 216(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
-; AVX512F-NEXT: vptestmd %zmm5, %zmm4, %k2
-; AVX512F-NEXT: vmovd %edi, %xmm4
-; AVX512F-NEXT: vpinsrb $1, %esi, %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $2, %edx, %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $3, %ecx, %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $4, %r8d, %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $5, %r9d, %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $6, 16(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $7, 24(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $8, 32(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $9, 40(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $10, 48(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $11, 56(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $12, 64(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $13, 72(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $14, 80(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpinsrb $15, 88(%rbp), %xmm4, %xmm4
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
-; AVX512F-NEXT: vptestmd %zmm5, %zmm4, %k4
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero,xmm4[4],zero,zero,zero,xmm4[5],zero,zero,zero,xmm4[6],zero,zero,zero,xmm4[7],zero,zero,zero,xmm4[8],zero,zero,zero,xmm4[9],zero,zero,zero,xmm4[10],zero,zero,zero,xmm4[11],zero,zero,zero,xmm4[12],zero,zero,zero,xmm4[13],zero,zero,zero,xmm4[14],zero,zero,zero,xmm4[15],zero,zero,zero
+; AVX512F-NEXT: vpbroadcastd {{.*#+}} zmm4 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512F-NEXT: vptestmd %zmm4, %zmm5, %k1
+; AVX512F-NEXT: vmovd {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vpinsrb $1, 232(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $2, 240(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $3, 248(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $4, 256(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $5, 264(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $6, 272(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $7, 280(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $8, 288(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $9, 296(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $10, 304(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $11, 312(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $12, 320(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $13, 328(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $14, 336(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $15, 344(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
+; AVX512F-NEXT: vptestmd %zmm4, %zmm5, %k2
+; AVX512F-NEXT: vmovd {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; AVX512F-NEXT: vpinsrb $1, 104(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $2, 112(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $3, 120(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $4, 128(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $5, 136(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $6, 144(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $7, 152(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $8, 160(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $9, 168(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $10, 176(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $11, 184(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $12, 192(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $13, 200(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $14, 208(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $15, 216(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
+; AVX512F-NEXT: vptestmd %zmm4, %zmm5, %k3
+; AVX512F-NEXT: vmovd %edi, %xmm5
+; AVX512F-NEXT: vpinsrb $1, %esi, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $2, %edx, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $3, %ecx, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $4, %r8d, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $5, %r9d, %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $6, 16(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $7, 24(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $8, 32(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $9, 40(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $10, 48(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $11, 56(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $12, 64(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $13, 72(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $14, 80(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpinsrb $15, 88(%rbp), %xmm5, %xmm5
+; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm5 = xmm5[0],zero,zero,zero,xmm5[1],zero,zero,zero,xmm5[2],zero,zero,zero,xmm5[3],zero,zero,zero,xmm5[4],zero,zero,zero,xmm5[5],zero,zero,zero,xmm5[6],zero,zero,zero,xmm5[7],zero,zero,zero,xmm5[8],zero,zero,zero,xmm5[9],zero,zero,zero,xmm5[10],zero,zero,zero,xmm5[11],zero,zero,zero,xmm5[12],zero,zero,zero,xmm5[13],zero,zero,zero,xmm5[14],zero,zero,zero,xmm5[15],zero,zero,zero
+; AVX512F-NEXT: vptestmd %zmm4, %zmm5, %k4
; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k4} {z}
; AVX512F-NEXT: vmovdqa64 %zmm0, (%rsp)
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k4} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm0, %zmm4
-; AVX512F-NEXT: vextracti64x4 $1, %zmm4, %ymm5
-; AVX512F-NEXT: vpaddd %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
-; AVX512F-NEXT: vmovd %xmm4, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $31, %ecx
-; AVX512F-NEXT: vpcompressd %zmm1, %zmm1 {%k2} {z}
-; AVX512F-NEXT: vmovdqa64 %zmm1, (%rsp,%rcx,4)
-; AVX512F-NEXT: vpcompressd %zmm2, %zmm1 {%k3} {z}
+; AVX512F-NEXT: vmovdqa32 %zmm4, %zmm0 {%k4} {z}
+; AVX512F-NEXT: vpmovdb %zmm0, %xmm5
+; AVX512F-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512F-NEXT: vpsadbw %xmm0, %xmm5, %xmm5
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
+; AVX512F-NEXT: vpaddq %xmm6, %xmm5, %xmm5
+; AVX512F-NEXT: vmovq %xmm5, %rax
+; AVX512F-NEXT: andl $31, %eax
+; AVX512F-NEXT: vpcompressd %zmm1, %zmm1 {%k3} {z}
+; AVX512F-NEXT: vmovdqa64 %zmm1, (%rsp,%rax,4)
+; AVX512F-NEXT: vpcompressd %zmm2, %zmm1 {%k2} {z}
; AVX512F-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k3} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm1, %zmm1
-; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512F-NEXT: vpaddd %ymm2, %ymm1, %ymm1
-; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT: vmovdqa32 %zmm4, %zmm1 {%k2} {z}
+; AVX512F-NEXT: vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT: vpsadbw %xmm0, %xmm1, %xmm1
; AVX512F-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT: vpextrd $1, %xmm1, %eax
-; AVX512F-NEXT: vmovd %xmm1, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vpaddq %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT: vmovq %xmm1, %rax
+; AVX512F-NEXT: andl $31, %eax
; AVX512F-NEXT: vpcompressd %zmm3, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovdqa64 %zmm1, 128(%rsp,%rcx,4)
-; AVX512F-NEXT: vmovdqa64 (%rsp), %zmm1
+; AVX512F-NEXT: vmovdqa64 %zmm1, 128(%rsp,%rax,4)
+; AVX512F-NEXT: vmovaps (%rsp), %zmm1
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm2
-; AVX512F-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k2} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm1, %zmm1
-; AVX512F-NEXT: vpsubd %zmm0, %zmm1, %zmm0
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vmovaps %zmm1, {{[0-9]+}}(%rsp)
+; AVX512F-NEXT: vmovdqa32 %zmm4, %zmm1 {%k3} {z}
+; AVX512F-NEXT: vpmovdb %zmm1, %xmm1
+; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k4} {z} = -1
+; AVX512F-NEXT: vpmovdb %zmm3, %xmm3
+; AVX512F-NEXT: vpsubb %xmm3, %xmm1, %xmm1
+; AVX512F-NEXT: vpsadbw %xmm0, %xmm1, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512F-NEXT: vmovd %xmm0, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $63, %ecx
+; AVX512F-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vmovq %xmm0, %rax
+; AVX512F-NEXT: andl $63, %eax
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm0
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
-; AVX512F-NEXT: vmovaps %zmm0, 256(%rsp,%rcx,4)
+; AVX512F-NEXT: vmovaps %zmm0, 256(%rsp,%rax,4)
; AVX512F-NEXT: vmovaps %zmm2, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vmovaps %zmm1, 320(%rsp,%rcx,4)
+; AVX512F-NEXT: vmovaps %zmm1, 320(%rsp,%rax,4)
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm0
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm2
@@ -3418,63 +3372,48 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512VL-NEXT: andq $-64, %rsp
; AVX512VL-NEXT: subq $576, %rsp # imm = 0x240
; AVX512VL-NEXT: vpsllw $7, %zmm0, %zmm0
-; AVX512VL-NEXT: vpmovb2m %zmm0, %k2
-; AVX512VL-NEXT: kshiftrq $48, %k2, %k1
-; AVX512VL-NEXT: kshiftrq $32, %k2, %k3
-; AVX512VL-NEXT: vpcompressd %zmm1, %zmm0 {%k2} {z}
+; AVX512VL-NEXT: vpmovb2m %zmm0, %k1
+; AVX512VL-NEXT: kshiftrq $48, %k1, %k2
+; AVX512VL-NEXT: kshiftrq $32, %k1, %k3
+; AVX512VL-NEXT: vpcompressd %zmm1, %zmm0 {%k1} {z}
; AVX512VL-NEXT: vmovdqa64 %zmm0, (%rsp)
-; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1
-; AVX512VL-NEXT: vpsrld $31, %zmm0, %zmm1
-; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm5
-; AVX512VL-NEXT: vpaddd %ymm5, %ymm1, %ymm1
-; AVX512VL-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX512VL-NEXT: vpaddd %xmm5, %xmm1, %xmm1
-; AVX512VL-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
-; AVX512VL-NEXT: vpaddd %xmm5, %xmm1, %xmm1
-; AVX512VL-NEXT: vpextrd $1, %xmm1, %eax
-; AVX512VL-NEXT: vmovd %xmm1, %ecx
-; AVX512VL-NEXT: addl %eax, %ecx
-; AVX512VL-NEXT: andl $31, %ecx
-; AVX512VL-NEXT: kshiftrq $16, %k2, %k2
-; AVX512VL-NEXT: vpcompressd %zmm2, %zmm1 {%k2} {z}
-; AVX512VL-NEXT: vmovdqa64 %zmm1, (%rsp,%rcx,4)
+; AVX512VL-NEXT: vpbroadcastb {{.*#+}} xmm0 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT: vmovdqu8 %xmm0, %xmm1 {%k1} {z}
+; AVX512VL-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX512VL-NEXT: vpsadbw %xmm5, %xmm1, %xmm1
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm6, %xmm1, %xmm1
+; AVX512VL-NEXT: vmovq %xmm1, %rax
+; AVX512VL-NEXT: andl $31, %eax
+; AVX512VL-NEXT: kshiftrq $16, %k1, %k4
+; AVX512VL-NEXT: vpcompressd %zmm2, %zmm1 {%k4} {z}
+; AVX512VL-NEXT: vmovdqa64 %zmm1, (%rsp,%rax,4)
; AVX512VL-NEXT: vpcompressd %zmm3, %zmm1 {%k3} {z}
; AVX512VL-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 {%k3} {z} = -1
-; AVX512VL-NEXT: vpsrld $31, %zmm1, %zmm1
-; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512VL-NEXT: vpaddd %ymm2, %ymm1, %ymm1
-; AVX512VL-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX512VL-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT: vmovdqu8 %xmm0, %xmm1 {%k3} {z}
+; AVX512VL-NEXT: vpsadbw %xmm5, %xmm1, %xmm1
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; AVX512VL-NEXT: vpaddd %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT: vpextrd $1, %xmm1, %eax
-; AVX512VL-NEXT: vmovd %xmm1, %ecx
-; AVX512VL-NEXT: addl %eax, %ecx
-; AVX512VL-NEXT: andl $31, %ecx
-; AVX512VL-NEXT: vpcompressd %zmm4, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovdqa64 %zmm1, 128(%rsp,%rcx,4)
-; AVX512VL-NEXT: vmovdqa64 (%rsp), %zmm1
+; AVX512VL-NEXT: vpaddq %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT: vmovq %xmm1, %rax
+; AVX512VL-NEXT: andl $31, %eax
+; AVX512VL-NEXT: vpcompressd %zmm4, %zmm1 {%k2} {z}
+; AVX512VL-NEXT: vmovdqa64 %zmm1, 128(%rsp,%rax,4)
+; AVX512VL-NEXT: vmovaps (%rsp), %zmm1
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm2
-; AVX512VL-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 {%k2} {z} = -1
-; AVX512VL-NEXT: vpsrld $31, %zmm1, %zmm1
-; AVX512VL-NEXT: vpsubd %zmm0, %zmm1, %zmm0
-; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512VL-NEXT: vpaddd %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512VL-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovaps %zmm1, {{[0-9]+}}(%rsp)
+; AVX512VL-NEXT: vmovdqu8 %xmm0, %xmm0 {%k4} {z}
+; AVX512VL-NEXT: vpmovm2b %k1, %xmm1
+; AVX512VL-NEXT: vpsubb %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsadbw %xmm5, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512VL-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512VL-NEXT: vmovd %xmm0, %ecx
-; AVX512VL-NEXT: addl %eax, %ecx
-; AVX512VL-NEXT: andl $63, %ecx
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovq %xmm0, %rax
+; AVX512VL-NEXT: andl $63, %eax
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm0
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
-; AVX512VL-NEXT: vmovaps %zmm0, 256(%rsp,%rcx,4)
+; AVX512VL-NEXT: vmovaps %zmm0, 256(%rsp,%rax,4)
; AVX512VL-NEXT: vmovaps %zmm2, {{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovaps %zmm1, 320(%rsp,%rcx,4)
+; AVX512VL-NEXT: vmovaps %zmm1, 320(%rsp,%rax,4)
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm0
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm2
@@ -3507,20 +3446,27 @@ define <8 x i64> @test_compress_knownbits_zext_v8i16_8i64(<8 x i16> %vec, <8 x
; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [255,255,255,255]
+; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT: vpackusdw %ymm3, %ymm2, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
-; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: andl $7, %eax
-; AVX2-NEXT: vpextrw $1, %xmm1, %ecx
-; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vmovd %xmm2, %ecx
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vpextrw $1, %xmm1, %eax
+; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm1, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: addq %rdx, %rax
; AVX2-NEXT: vpextrw $2, %xmm1, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rcx, %rsi
+; AVX2-NEXT: addq %rax, %rsi
; AVX2-NEXT: vpextrw $3, %xmm1, %edi
; AVX2-NEXT: andl $1, %edi
; AVX2-NEXT: addq %rsi, %rdi
@@ -3539,11 +3485,11 @@ define <8 x i64> @test_compress_knownbits_zext_v8i16_8i64(<8 x i16> %vec, <8 x
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpextrq $1, %xmm1, %rbx
; AVX2-NEXT: cmpq $8, %r11
-; AVX2-NEXT: cmovbq (%rsp,%rax,8), %rbx
+; AVX2-NEXT: cmovbq (%rsp,%rcx,8), %rbx
; AVX2-NEXT: vmovq %xmm4, (%rsp)
; AVX2-NEXT: vpextrq $1, %xmm4, (%rsp,%rdx,8)
; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm2
-; AVX2-NEXT: vmovq %xmm2, (%rsp,%rcx,8)
+; AVX2-NEXT: vmovq %xmm2, (%rsp,%rax,8)
; AVX2-NEXT: vpextrq $1, %xmm2, (%rsp,%rsi,8)
; AVX2-NEXT: andl $7, %edi
; AVX2-NEXT: vmovq %xmm0, (%rsp,%rdi,8)
@@ -3611,20 +3557,27 @@ define <8 x i64> @test_compress_knownbits_sext_v8i16_8i64(<8 x i16> %vec, <8 x i
; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
; AVX2-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [255,255,255,255]
+; AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpackusdw %ymm3, %ymm2, %ymm2
+; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; AVX2-NEXT: vpackusdw %ymm3, %ymm2, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
-; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
-; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: addl %ecx, %eax
-; AVX2-NEXT: andl $7, %eax
-; AVX2-NEXT: vpextrw $1, %xmm1, %ecx
-; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,1,3]
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vmovd %xmm2, %ecx
+; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vpextrw $1, %xmm1, %eax
+; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm1, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: addq %rdx, %rax
; AVX2-NEXT: vpextrw $2, %xmm1, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rcx, %rsi
+; AVX2-NEXT: addq %rax, %rsi
; AVX2-NEXT: vpextrw $3, %xmm1, %edi
; AVX2-NEXT: andl $1, %edi
; AVX2-NEXT: addq %rsi, %rdi
@@ -3643,11 +3596,11 @@ define <8 x i64> @test_compress_knownbits_sext_v8i16_8i64(<8 x i16> %vec, <8 x i
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpextrq $1, %xmm1, %rbx
; AVX2-NEXT: cmpq $8, %r11
-; AVX2-NEXT: cmovbq (%rsp,%rax,8), %rbx
+; AVX2-NEXT: cmovbq (%rsp,%rcx,8), %rbx
; AVX2-NEXT: vmovq %xmm4, (%rsp)
; AVX2-NEXT: vpextrq $1, %xmm4, (%rsp,%rdx,8)
; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm2
-; AVX2-NEXT: vmovq %xmm2, (%rsp,%rcx,8)
+; AVX2-NEXT: vmovq %xmm2, (%rsp,%rax,8)
; AVX2-NEXT: vpextrq $1, %xmm2, (%rsp,%rsi,8)
; AVX2-NEXT: andl $7, %edi
; AVX2-NEXT: vmovq %xmm0, (%rsp,%rdi,8)
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll b/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
index 43bc6f5ef8429..33ae1e9513118 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
@@ -105,42 +105,61 @@ define i64 @test_v4i64_v4i16(<4 x i64> %a0) nounwind {
; X86-SSE2: # %bb.0:
; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE2-NEXT: paddq %xmm1, %xmm0
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm0, %xmm1
-; X86-SSE2-NEXT: movd %xmm1, %eax
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X86-SSE2-NEXT: movd %xmm0, %edx
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: xorl %edx, %edx
; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v4i64_v4i16:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; X64-SSE-NEXT: paddq %xmm1, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE-NEXT: paddq %xmm0, %xmm1
-; X64-SSE-NEXT: movq %xmm1, %rax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v4i64_v4i16:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v4i64_v4i16:
; X86-SSE4: # %bb.0:
; X86-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
; X86-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE4-NEXT: paddq %xmm1, %xmm0
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm0, %xmm1
-; X86-SSE4-NEXT: movd %xmm1, %eax
-; X86-SSE4-NEXT: pextrd $1, %xmm1, %edx
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: pextrd $1, %xmm0, %edx
; X86-SSE4-NEXT: retl
;
+; X64-SSE4-LABEL: test_v4i64_v4i16:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; X64-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X64-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: retq
+;
; X86-AVX1-LABEL: test_v4i64_v4i16:
; X86-AVX1: # %bb.0:
; X86-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: vpextrd $1, %xmm0, %edx
; X86-AVX1-NEXT: vzeroupper
@@ -150,9 +169,11 @@ define i64 @test_v4i64_v4i16(<4 x i64> %a0) nounwind {
; X64-AVX1: # %bb.0:
; X64-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
; X64-AVX1-NEXT: vzeroupper
; X64-AVX1-NEXT: retq
@@ -160,10 +181,15 @@ define i64 @test_v4i64_v4i16(<4 x i64> %a0) nounwind {
; X86-AVX2-LABEL: test_v4i64_v4i16:
; X86-AVX2: # %bb.0:
; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: vpextrd $1, %xmm0, %edx
; X86-AVX2-NEXT: vzeroupper
@@ -172,35 +198,28 @@ define i64 @test_v4i64_v4i16(<4 x i64> %a0) nounwind {
; X64-AVX2-LABEL: test_v4i64_v4i16:
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovq %xmm0, %rax
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
-; AVX512BW-LABEL: test_v4i64_v4i16:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512BW-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vmovq %xmm0, %rax
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
-;
-; AVX512BWVL-LABEL: test_v4i64_v4i16:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512BWVL-NEXT: vpmovqb %ymm0, %xmm0
-; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vmovq %xmm0, %rax
-; AVX512BWVL-NEXT: vzeroupper
-; AVX512BWVL-NEXT: retq
+; AVX512-LABEL: test_v4i64_v4i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%1 = and <4 x i64> %a0, <i64 15, i64 31, i64 63, i64 127>
%2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %1)
ret i64 %2
@@ -214,35 +233,36 @@ define i64 @test_v8i64_v8i8(<8 x i64> %a0) nounwind {
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE2-NEXT: psrlq $60, %xmm2
+; X86-SSE2-NEXT: psrlq $60, %xmm1
; X86-SSE2-NEXT: psrlq $60, %xmm0
-; X86-SSE2-NEXT: paddq %xmm2, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
; X86-SSE2-NEXT: psrlq $60, %xmm3
-; X86-SSE2-NEXT: psrlq $60, %xmm1
-; X86-SSE2-NEXT: paddq %xmm3, %xmm1
-; X86-SSE2-NEXT: paddq %xmm0, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm1, %xmm0
+; X86-SSE2-NEXT: psrlq $60, %xmm2
+; X86-SSE2-NEXT: packuswb %xmm3, %xmm2
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
; X86-SSE2-NEXT: movd %xmm0, %eax
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
-; X86-SSE2-NEXT: movd %xmm0, %edx
+; X86-SSE2-NEXT: xorl %edx, %edx
; X86-SSE2-NEXT: movl %ebp, %esp
; X86-SSE2-NEXT: popl %ebp
; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v8i64_v8i8:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: psrlq $60, %xmm2
-; X64-SSE-NEXT: psrlq $60, %xmm0
-; X64-SSE-NEXT: paddq %xmm2, %xmm0
-; X64-SSE-NEXT: psrlq $60, %xmm3
-; X64-SSE-NEXT: psrlq $60, %xmm1
-; X64-SSE-NEXT: paddq %xmm3, %xmm1
-; X64-SSE-NEXT: paddq %xmm0, %xmm1
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X64-SSE-NEXT: paddq %xmm1, %xmm0
-; X64-SSE-NEXT: movq %xmm0, %rax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v8i64_v8i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: psrlq $60, %xmm1
+; X64-SSE2-NEXT: psrlq $60, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: psrlq $60, %xmm3
+; X64-SSE2-NEXT: psrlq $60, %xmm2
+; X64-SSE2-NEXT: packuswb %xmm3, %xmm2
+; X64-SSE2-NEXT: packuswb %xmm2, %xmm0
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v8i64_v8i8:
; X86-SSE4: # %bb.0:
@@ -251,34 +271,51 @@ define i64 @test_v8i64_v8i8(<8 x i64> %a0) nounwind {
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE4-NEXT: psrlq $60, %xmm2
+; X86-SSE4-NEXT: psrlq $60, %xmm1
; X86-SSE4-NEXT: psrlq $60, %xmm0
-; X86-SSE4-NEXT: paddq %xmm2, %xmm0
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm0
; X86-SSE4-NEXT: psrlq $60, %xmm3
-; X86-SSE4-NEXT: psrlq $60, %xmm1
-; X86-SSE4-NEXT: paddq %xmm3, %xmm1
-; X86-SSE4-NEXT: paddq %xmm0, %xmm1
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm1, %xmm0
+; X86-SSE4-NEXT: psrlq $60, %xmm2
+; X86-SSE4-NEXT: packusdw %xmm3, %xmm2
+; X86-SSE4-NEXT: packusdw %xmm2, %xmm0
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm0
; X86-SSE4-NEXT: movd %xmm0, %eax
; X86-SSE4-NEXT: pextrd $1, %xmm0, %edx
; X86-SSE4-NEXT: movl %ebp, %esp
; X86-SSE4-NEXT: popl %ebp
; X86-SSE4-NEXT: retl
;
+; X64-SSE4-LABEL: test_v8i64_v8i8:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: psrlq $60, %xmm1
+; X64-SSE4-NEXT: psrlq $60, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X64-SSE4-NEXT: psrlq $60, %xmm3
+; X64-SSE4-NEXT: psrlq $60, %xmm2
+; X64-SSE4-NEXT: packusdw %xmm3, %xmm2
+; X64-SSE4-NEXT: packusdw %xmm2, %xmm0
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: retq
+;
; X86-AVX1-LABEL: test_v8i64_v8i8:
; X86-AVX1: # %bb.0:
-; X86-AVX1-NEXT: vpsrlq $60, %xmm1, %xmm2
-; X86-AVX1-NEXT: vpsrlq $60, %xmm0, %xmm3
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-NEXT: vpsrlq $60, %xmm1, %xmm1
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT: vpsrlq $60, %xmm2, %xmm2
; X86-AVX1-NEXT: vpsrlq $60, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
+; X86-AVX1-NEXT: vpsrlq $60, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpsrlq $60, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: vpextrd $1, %xmm0, %edx
; X86-AVX1-NEXT: vzeroupper
@@ -286,17 +323,18 @@ define i64 @test_v8i64_v8i8(<8 x i64> %a0) nounwind {
;
; X64-AVX1-LABEL: test_v8i64_v8i8:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vpsrlq $60, %xmm1, %xmm2
-; X64-AVX1-NEXT: vpsrlq $60, %xmm0, %xmm3
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X64-AVX1-NEXT: vpsrlq $60, %xmm1, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; X64-AVX1-NEXT: vpsrlq $60, %xmm2, %xmm2
; X64-AVX1-NEXT: vpsrlq $60, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
+; X64-AVX1-NEXT: vpsrlq $60, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpsrlq $60, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
; X64-AVX1-NEXT: vzeroupper
; X64-AVX1-NEXT: retq
@@ -305,11 +343,15 @@ define i64 @test_v8i64_v8i8(<8 x i64> %a0) nounwind {
; X86-AVX2: # %bb.0:
; X86-AVX2-NEXT: vpsrlq $60, %ymm1, %ymm1
; X86-AVX2-NEXT: vpsrlq $60, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: vpextrd $1, %xmm0, %edx
; X86-AVX2-NEXT: vzeroupper
@@ -319,11 +361,15 @@ define i64 @test_v8i64_v8i8(<8 x i64> %a0) nounwind {
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpsrlq $60, %ymm1, %ymm1
; X64-AVX2-NEXT: vpsrlq $60, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovq %xmm0, %rax
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
@@ -349,28 +395,31 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: movl %esp, %ebp
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1,0,1,0]
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm2
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm4
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm4
; X86-SSE2-NEXT: pand %xmm3, %xmm4
-; X86-SSE2-NEXT: paddq %xmm1, %xmm4
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm5
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm5
; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: paddq %xmm5, %xmm1
-; X86-SSE2-NEXT: paddq %xmm4, %xmm1
+; X86-SSE2-NEXT: packuswb %xmm4, %xmm5
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm6
+; X86-SSE2-NEXT: pand %xmm3, %xmm6
; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm4
; X86-SSE2-NEXT: pand %xmm3, %xmm4
-; X86-SSE2-NEXT: paddq %xmm0, %xmm4
-; X86-SSE2-NEXT: pand 56(%ebp), %xmm3
-; X86-SSE2-NEXT: paddq %xmm2, %xmm3
-; X86-SSE2-NEXT: paddq %xmm4, %xmm3
-; X86-SSE2-NEXT: paddq %xmm1, %xmm3
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm3, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm6, %xmm4
+; X86-SSE2-NEXT: packuswb %xmm5, %xmm4
+; X86-SSE2-NEXT: pand %xmm3, %xmm1
+; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: pand 8(%ebp), %xmm3
+; X86-SSE2-NEXT: packuswb %xmm3, %xmm2
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm4, %xmm0
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE2-NEXT: paddq %xmm1, %xmm0
; X86-SSE2-NEXT: movd %xmm0, %eax
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; X86-SSE2-NEXT: movd %xmm0, %edx
@@ -380,24 +429,27 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
;
; X64-SSE2-LABEL: test_v16i64_v16i8:
; X64-SSE2: # %bb.0:
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm8 = [1,1]
-; X64-SSE2-NEXT: pand %xmm8, %xmm5
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
-; X64-SSE2-NEXT: paddq %xmm5, %xmm1
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm8 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
; X64-SSE2-NEXT: pand %xmm8, %xmm7
-; X64-SSE2-NEXT: pand %xmm8, %xmm3
-; X64-SSE2-NEXT: paddq %xmm7, %xmm3
-; X64-SSE2-NEXT: paddq %xmm1, %xmm3
-; X64-SSE2-NEXT: pand %xmm8, %xmm4
-; X64-SSE2-NEXT: pand %xmm8, %xmm0
-; X64-SSE2-NEXT: paddq %xmm4, %xmm0
; X64-SSE2-NEXT: pand %xmm8, %xmm6
+; X64-SSE2-NEXT: packuswb %xmm7, %xmm6
+; X64-SSE2-NEXT: pand %xmm8, %xmm5
+; X64-SSE2-NEXT: pand %xmm8, %xmm4
+; X64-SSE2-NEXT: packuswb %xmm5, %xmm4
+; X64-SSE2-NEXT: packuswb %xmm6, %xmm4
+; X64-SSE2-NEXT: pand %xmm8, %xmm3
; X64-SSE2-NEXT: pand %xmm8, %xmm2
-; X64-SSE2-NEXT: paddq %xmm6, %xmm2
-; X64-SSE2-NEXT: paddq %xmm0, %xmm2
-; X64-SSE2-NEXT: paddq %xmm3, %xmm2
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm2, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm3, %xmm2
+; X64-SSE2-NEXT: pand %xmm8, %xmm1
+; X64-SSE2-NEXT: pand %xmm8, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm2, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm4, %xmm0
+; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE2-NEXT: paddq %xmm1, %xmm0
; X64-SSE2-NEXT: movq %xmm0, %rax
; X64-SSE2-NEXT: retq
;
@@ -407,28 +459,31 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X86-SSE4-NEXT: movl %esp, %ebp
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
-; X86-SSE4-NEXT: pmovsxbq {{.*#+}} xmm3 = [1,1]
-; X86-SSE4-NEXT: pand %xmm3, %xmm1
-; X86-SSE4-NEXT: pand %xmm3, %xmm0
-; X86-SSE4-NEXT: pand %xmm3, %xmm2
-; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm4
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm3 = [255,255]
+; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm4
; X86-SSE4-NEXT: pand %xmm3, %xmm4
-; X86-SSE4-NEXT: paddq %xmm1, %xmm4
-; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm5
+; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm5
; X86-SSE4-NEXT: pand %xmm3, %xmm5
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm1
-; X86-SSE4-NEXT: pand %xmm3, %xmm1
-; X86-SSE4-NEXT: paddq %xmm5, %xmm1
-; X86-SSE4-NEXT: paddq %xmm4, %xmm1
+; X86-SSE4-NEXT: packusdw %xmm4, %xmm5
+; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm6
+; X86-SSE4-NEXT: pand %xmm3, %xmm6
; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm4
; X86-SSE4-NEXT: pand %xmm3, %xmm4
-; X86-SSE4-NEXT: paddq %xmm0, %xmm4
-; X86-SSE4-NEXT: pand 56(%ebp), %xmm3
-; X86-SSE4-NEXT: paddq %xmm2, %xmm3
-; X86-SSE4-NEXT: paddq %xmm4, %xmm3
-; X86-SSE4-NEXT: paddq %xmm1, %xmm3
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm3, %xmm0
+; X86-SSE4-NEXT: packusdw %xmm6, %xmm4
+; X86-SSE4-NEXT: packusdw %xmm5, %xmm4
+; X86-SSE4-NEXT: pand %xmm3, %xmm1
+; X86-SSE4-NEXT: pand %xmm3, %xmm0
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X86-SSE4-NEXT: pand %xmm3, %xmm2
+; X86-SSE4-NEXT: pand 8(%ebp), %xmm3
+; X86-SSE4-NEXT: packusdw %xmm3, %xmm2
+; X86-SSE4-NEXT: packusdw %xmm2, %xmm0
+; X86-SSE4-NEXT: packuswb %xmm4, %xmm0
+; X86-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm1, %xmm0
; X86-SSE4-NEXT: movd %xmm0, %eax
; X86-SSE4-NEXT: pextrd $1, %xmm0, %edx
; X86-SSE4-NEXT: movl %ebp, %esp
@@ -437,24 +492,27 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
;
; X64-SSE4-LABEL: test_v16i64_v16i8:
; X64-SSE4: # %bb.0:
-; X64-SSE4-NEXT: pmovsxbq {{.*#+}} xmm8 = [1,1]
-; X64-SSE4-NEXT: pand %xmm8, %xmm5
-; X64-SSE4-NEXT: pand %xmm8, %xmm1
-; X64-SSE4-NEXT: paddq %xmm5, %xmm1
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm8 = [255,255]
; X64-SSE4-NEXT: pand %xmm8, %xmm7
-; X64-SSE4-NEXT: pand %xmm8, %xmm3
-; X64-SSE4-NEXT: paddq %xmm7, %xmm3
-; X64-SSE4-NEXT: paddq %xmm1, %xmm3
-; X64-SSE4-NEXT: pand %xmm8, %xmm4
-; X64-SSE4-NEXT: pand %xmm8, %xmm0
-; X64-SSE4-NEXT: paddq %xmm4, %xmm0
; X64-SSE4-NEXT: pand %xmm8, %xmm6
+; X64-SSE4-NEXT: packusdw %xmm7, %xmm6
+; X64-SSE4-NEXT: pand %xmm8, %xmm5
+; X64-SSE4-NEXT: pand %xmm8, %xmm4
+; X64-SSE4-NEXT: packusdw %xmm5, %xmm4
+; X64-SSE4-NEXT: packusdw %xmm6, %xmm4
+; X64-SSE4-NEXT: pand %xmm8, %xmm3
; X64-SSE4-NEXT: pand %xmm8, %xmm2
-; X64-SSE4-NEXT: paddq %xmm6, %xmm2
-; X64-SSE4-NEXT: paddq %xmm0, %xmm2
-; X64-SSE4-NEXT: paddq %xmm3, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm2, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm3, %xmm2
+; X64-SSE4-NEXT: pand %xmm8, %xmm1
+; X64-SSE4-NEXT: pand %xmm8, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm2, %xmm0
+; X64-SSE4-NEXT: packuswb %xmm4, %xmm0
+; X64-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm1, %xmm0
; X64-SSE4-NEXT: movq %xmm0, %rax
; X64-SSE4-NEXT: retq
;
@@ -464,22 +522,25 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X86-AVX1-NEXT: movl %esp, %ebp
; X86-AVX1-NEXT: andl $-32, %esp
; X86-AVX1-NEXT: subl $32, %esp
-; X86-AVX1-NEXT: vbroadcastsd {{.*#+}} ymm3 = [1,0,1,0,1,0,1,0]
+; X86-AVX1-NEXT: vbroadcastsd {{.*#+}} ymm3 = [255,0,255,0,255,0,255,0]
; X86-AVX1-NEXT: vandps %ymm3, %ymm1, %ymm1
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-NEXT: vpackusdw %xmm4, %xmm1, %xmm1
; X86-AVX1-NEXT: vandps %ymm3, %ymm0, %ymm0
-; X86-AVX1-NEXT: vandps %ymm3, %ymm2, %ymm2
-; X86-AVX1-NEXT: vandps 8(%ebp), %ymm3, %ymm3
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm4
-; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm2
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-NEXT: vpackusdw %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vandps %ymm3, %ymm2, %ymm1
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
+; X86-AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT: vandps 8(%ebp), %ymm3, %ymm2
+; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
@@ -491,22 +552,25 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
;
; X64-AVX1-LABEL: test_v16i64_v16i8:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vbroadcastsd {{.*#+}} ymm4 = [1,1,1,1]
-; X64-AVX1-NEXT: vandps %ymm4, %ymm0, %ymm0
+; X64-AVX1-NEXT: vbroadcastsd {{.*#+}} ymm4 = [255,255,255,255]
+; X64-AVX1-NEXT: vandps %ymm4, %ymm3, %ymm3
+; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm5
+; X64-AVX1-NEXT: vpackusdw %xmm5, %xmm3, %xmm3
; X64-AVX1-NEXT: vandps %ymm4, %ymm2, %ymm2
+; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-NEXT: vpackusdw %xmm5, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
; X64-AVX1-NEXT: vandps %ymm4, %ymm1, %ymm1
-; X64-AVX1-NEXT: vandps %ymm4, %ymm3, %ymm3
-; X64-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm4
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm5
-; X64-AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X64-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm0, %xmm4, %xmm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; X64-AVX1-NEXT: vpackusdw %xmm3, %xmm1, %xmm1
+; X64-AVX1-NEXT: vandps %ymm4, %ymm0, %ymm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; X64-AVX1-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
@@ -519,16 +583,22 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X86-AVX2-NEXT: movl %esp, %ebp
; X86-AVX2-NEXT: andl $-32, %esp
; X86-AVX2-NEXT: subl $32, %esp
-; X86-AVX2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [1,0,1,0,1,0,1,0]
+; X86-AVX2-NEXT: vpbroadcastq {{.*#+}} ymm3 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
; X86-AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpaddq %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm1
; X86-AVX2-NEXT: vpand 8(%ebp), %ymm3, %ymm2
-; X86-AVX2-NEXT: vpaddq %ymm2, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackusdw %ymm2, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
@@ -540,16 +610,22 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
;
; X64-AVX2-LABEL: test_v16i64_v16i8:
; X64-AVX2: # %bb.0:
-; X64-AVX2-NEXT: vpbroadcastq {{.*#+}} ymm4 = [1,1,1,1]
+; X64-AVX2-NEXT: vpbroadcastq {{.*#+}} ymm4 = [255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0,255,0,0,0,0,0,0,0]
+; X64-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
; X64-AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2
-; X64-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpaddq %ymm2, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm2
+; X64-AVX2-NEXT: vpackusdw %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
; X64-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
-; X64-AVX2-NEXT: vpaddq %ymm2, %ymm1, %ymm1
-; X64-AVX2-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm2, %ymm0, %ymm0
; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovq %xmm0, %rax
@@ -593,74 +669,108 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
;
define i32 @test_v2i32_v2i16(<2 x i32> %a0) nounwind {
-; X86-SSE-LABEL: test_v2i32_v2i16:
-; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-SSE-NEXT: paddd %xmm0, %xmm1
-; X86-SSE-NEXT: movd %xmm1, %eax
-; X86-SSE-NEXT: retl
+; X86-SSE2-LABEL: test_v2i32_v2i16:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v2i32_v2i16:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-SSE-NEXT: paddd %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v2i32_v2i16:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: retq
;
-; X86-AVX1-SLOW-LABEL: test_v2i32_v2i16:
-; X86-AVX1-SLOW: # %bb.0:
-; X86-AVX1-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-SLOW-NEXT: retl
+; X86-SSE4-LABEL: test_v2i32_v2i16:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: retl
;
-; X64-AVX1-SLOW-LABEL: test_v2i32_v2i16:
-; X64-AVX1-SLOW: # %bb.0:
-; X64-AVX1-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-SLOW-NEXT: retq
+; X64-SSE4-LABEL: test_v2i32_v2i16:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X64-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE4-NEXT: movd %xmm0, %eax
+; X64-SSE4-NEXT: retq
;
-; X86-AVX1-FAST-LABEL: test_v2i32_v2i16:
-; X86-AVX1-FAST: # %bb.0:
-; X86-AVX1-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-FAST-NEXT: retl
+; X86-AVX1-LABEL: test_v2i32_v2i16:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; X86-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: vzeroupper
+; X86-AVX1-NEXT: retl
;
-; X64-AVX1-FAST-LABEL: test_v2i32_v2i16:
-; X64-AVX1-FAST: # %bb.0:
-; X64-AVX1-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-FAST-NEXT: retq
+; X64-AVX1-LABEL: test_v2i32_v2i16:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; X64-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-NEXT: vzeroupper
+; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: test_v2i32_v2i16:
; X86-AVX2: # %bb.0:
-; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: vzeroupper
; X86-AVX2-NEXT: retl
;
; X64-AVX2-LABEL: test_v2i32_v2i16:
; X64-AVX2: # %bb.0:
-; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovd %xmm0, %eax
+; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
; AVX512-LABEL: test_v2i32_v2i16:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%1 = and <2 x i32> %a0, <i32 255, i32 255>
%2 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %1)
@@ -668,148 +778,143 @@ define i32 @test_v2i32_v2i16(<2 x i32> %a0) nounwind {
}
define i32 @test_v4i32(<4 x i32> %a0) nounwind {
-; SSE-LABEL: test_v4i32:
-; SSE: # %bb.0:
-; SSE-NEXT: psrld $31, %xmm0
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE-NEXT: paddd %xmm0, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; SSE-NEXT: paddd %xmm1, %xmm0
-; SSE-NEXT: movd %xmm0, %eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX1-SLOW-LABEL: test_v4i32:
-; AVX1-SLOW: # %bb.0:
-; AVX1-SLOW-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; AVX1-SLOW-NEXT: ret{{[l|q]}}
+; SSE2-LABEL: test_v4i32:
+; SSE2: # %bb.0:
+; SSE2-NEXT: psrld $31, %xmm0
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: packuswb %xmm1, %xmm0
+; SSE2-NEXT: packuswb %xmm1, %xmm0
+; SSE2-NEXT: psadbw %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: ret{{[l|q]}}
+;
+; SSE4-LABEL: test_v4i32:
+; SSE4: # %bb.0:
+; SSE4-NEXT: psrld $31, %xmm0
+; SSE4-NEXT: pxor %xmm1, %xmm1
+; SSE4-NEXT: packusdw %xmm1, %xmm0
+; SSE4-NEXT: packuswb %xmm1, %xmm0
+; SSE4-NEXT: psadbw %xmm1, %xmm0
+; SSE4-NEXT: movd %xmm0, %eax
+; SSE4-NEXT: ret{{[l|q]}}
;
-; AVX1-FAST-LABEL: test_v4i32:
-; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; AVX1-FAST-NEXT: ret{{[l|q]}}
+; AVX1-LABEL: test_v4i32:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpsrld $31, %xmm0, %xmm0
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vmovd %xmm0, %eax
+; AVX1-NEXT: ret{{[l|q]}}
;
; AVX2-LABEL: test_v4i32:
; AVX2: # %bb.0:
; AVX2-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: ret{{[l|q]}}
;
-; AVX512BW-LABEL: test_v4i32:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vmovd %xmm0, %eax
-; AVX512BW-NEXT: retq
-;
-; AVX512BWVL-LABEL: test_v4i32:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpmovdb %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vmovd %xmm0, %eax
-; AVX512BWVL-NEXT: retq
+; AVX512-LABEL: test_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpsrld $31, %xmm0, %xmm0
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%1 = lshr <4 x i32> %a0, <i32 31, i32 31, i32 31, i32 31>
%2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)
ret i32 %2
}
define i32 @test_v8i32_v8i8(<8 x i32> %a0) nounwind {
-; X86-SSE-LABEL: test_v8i32_v8i8:
-; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
-; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: por %xmm1, %xmm0
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE-NEXT: paddd %xmm0, %xmm1
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X86-SSE-NEXT: paddd %xmm1, %xmm0
-; X86-SSE-NEXT: movd %xmm0, %eax
-; X86-SSE-NEXT: retl
+; X86-SSE2-LABEL: test_v8i32_v8i8:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v8i32_v8i8:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; X64-SSE-NEXT: por %xmm1, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE-NEXT: paddd %xmm0, %xmm1
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X64-SSE-NEXT: paddd %xmm1, %xmm0
-; X64-SSE-NEXT: movd %xmm0, %eax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v8i32_v8i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: retq
;
-; X86-AVX1-SLOW-LABEL: test_v8i32_v8i8:
-; X86-AVX1-SLOW: # %bb.0:
-; X86-AVX1-SLOW-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X86-AVX1-SLOW-NEXT: vorps %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-SLOW-NEXT: vzeroupper
-; X86-AVX1-SLOW-NEXT: retl
+; X86-SSE4-LABEL: test_v8i32_v8i8:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: retl
;
-; X64-AVX1-SLOW-LABEL: test_v8i32_v8i8:
-; X64-AVX1-SLOW: # %bb.0:
-; X64-AVX1-SLOW-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X64-AVX1-SLOW-NEXT: vorps %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-SLOW-NEXT: vzeroupper
-; X64-AVX1-SLOW-NEXT: retq
+; X64-SSE4-LABEL: test_v8i32_v8i8:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; X64-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE4-NEXT: movd %xmm0, %eax
+; X64-SSE4-NEXT: retq
;
-; X86-AVX1-FAST-LABEL: test_v8i32_v8i8:
-; X86-AVX1-FAST: # %bb.0:
-; X86-AVX1-FAST-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
-; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-FAST-NEXT: vzeroupper
-; X86-AVX1-FAST-NEXT: retl
+; X86-AVX1-LABEL: test_v8i32_v8i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: vzeroupper
+; X86-AVX1-NEXT: retl
;
-; X64-AVX1-FAST-LABEL: test_v8i32_v8i8:
-; X64-AVX1-FAST: # %bb.0:
-; X64-AVX1-FAST-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
-; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-FAST-NEXT: vzeroupper
-; X64-AVX1-FAST-NEXT: retq
+; X64-AVX1-LABEL: test_v8i32_v8i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-NEXT: vzeroupper
+; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: test_v8i32_v8i8:
; X86-AVX2: # %bb.0:
; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X86-AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: vzeroupper
; X86-AVX2-NEXT: retl
@@ -817,35 +922,27 @@ define i32 @test_v8i32_v8i8(<8 x i32> %a0) nounwind {
; X64-AVX2-LABEL: test_v8i32_v8i8:
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X64-AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovd %xmm0, %eax
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
-; AVX512BW-LABEL: test_v8i32_v8i8:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512BW-NEXT: vpmovdb %zmm0, %xmm0
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vmovd %xmm0, %eax
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
-;
-; AVX512BWVL-LABEL: test_v8i32_v8i8:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512BWVL-NEXT: vpmovdb %ymm0, %xmm0
-; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vmovd %xmm0, %eax
-; AVX512BWVL-NEXT: vzeroupper
-; AVX512BWVL-NEXT: retq
+; AVX512-LABEL: test_v8i32_v8i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%1 = and <8 x i32> %a0, <i32 0, i32 1, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64>
%2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
ret i32 %2
@@ -860,17 +957,17 @@ define i32 @test_v16i32_v16i8(<16 x i32> %a0) nounwind {
; X86-SSE2-NEXT: subl $16, %esp
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm2
; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: paddd %xmm2, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
; X86-SSE2-NEXT: pand 8(%ebp), %xmm3
-; X86-SSE2-NEXT: paddd %xmm1, %xmm3
-; X86-SSE2-NEXT: paddd %xmm0, %xmm3
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
-; X86-SSE2-NEXT: paddd %xmm3, %xmm0
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-SSE2-NEXT: paddd %xmm0, %xmm1
-; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: packuswb %xmm3, %xmm2
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE2-NEXT: paddq %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
; X86-SSE2-NEXT: movl %ebp, %esp
; X86-SSE2-NEXT: popl %ebp
; X86-SSE2-NEXT: retl
@@ -878,18 +975,19 @@ define i32 @test_v16i32_v16i8(<16 x i32> %a0) nounwind {
; X64-SSE2-LABEL: test_v16i32_v16i8:
; X64-SSE2: # %bb.0:
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; X64-SSE2-NEXT: pand %xmm4, %xmm2
-; X64-SSE2-NEXT: pand %xmm4, %xmm0
-; X64-SSE2-NEXT: paddd %xmm2, %xmm0
; X64-SSE2-NEXT: pand %xmm4, %xmm3
+; X64-SSE2-NEXT: pand %xmm4, %xmm2
+; X64-SSE2-NEXT: packuswb %xmm3, %xmm2
; X64-SSE2-NEXT: pand %xmm4, %xmm1
-; X64-SSE2-NEXT: paddd %xmm3, %xmm1
-; X64-SSE2-NEXT: paddd %xmm0, %xmm1
+; X64-SSE2-NEXT: pand %xmm4, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm2, %xmm0
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X64-SSE2-NEXT: paddd %xmm1, %xmm0
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-SSE2-NEXT: paddd %xmm0, %xmm1
-; X64-SSE2-NEXT: movd %xmm1, %eax
+; X64-SSE2-NEXT: paddq %xmm1, %xmm0
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: # kill: def $eax killed $eax killed $rax
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v16i32_v16i8:
@@ -900,17 +998,17 @@ define i32 @test_v16i32_v16i8(<16 x i32> %a0) nounwind {
; X86-SSE4-NEXT: subl $16, %esp
; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm3 = [255,255,255,255]
; X86-SSE4-NEXT: pand %xmm3, %xmm1
-; X86-SSE4-NEXT: pand %xmm3, %xmm2
; X86-SSE4-NEXT: pand %xmm3, %xmm0
-; X86-SSE4-NEXT: paddd %xmm2, %xmm0
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X86-SSE4-NEXT: pand %xmm3, %xmm2
; X86-SSE4-NEXT: pand 8(%ebp), %xmm3
-; X86-SSE4-NEXT: paddd %xmm1, %xmm3
-; X86-SSE4-NEXT: paddd %xmm0, %xmm3
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
-; X86-SSE4-NEXT: paddd %xmm3, %xmm0
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-SSE4-NEXT: paddd %xmm0, %xmm1
-; X86-SSE4-NEXT: movd %xmm1, %eax
+; X86-SSE4-NEXT: packusdw %xmm3, %xmm2
+; X86-SSE4-NEXT: packuswb %xmm2, %xmm0
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
; X86-SSE4-NEXT: movl %ebp, %esp
; X86-SSE4-NEXT: popl %ebp
; X86-SSE4-NEXT: retl
@@ -918,70 +1016,92 @@ define i32 @test_v16i32_v16i8(<16 x i32> %a0) nounwind {
; X64-SSE4-LABEL: test_v16i32_v16i8:
; X64-SSE4: # %bb.0:
; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm4 = [255,255,255,255]
-; X64-SSE4-NEXT: pand %xmm4, %xmm2
-; X64-SSE4-NEXT: pand %xmm4, %xmm0
-; X64-SSE4-NEXT: paddd %xmm2, %xmm0
; X64-SSE4-NEXT: pand %xmm4, %xmm3
+; X64-SSE4-NEXT: pand %xmm4, %xmm2
+; X64-SSE4-NEXT: packusdw %xmm3, %xmm2
; X64-SSE4-NEXT: pand %xmm4, %xmm1
-; X64-SSE4-NEXT: paddd %xmm3, %xmm1
-; X64-SSE4-NEXT: paddd %xmm0, %xmm1
+; X64-SSE4-NEXT: pand %xmm4, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X64-SSE4-NEXT: packuswb %xmm2, %xmm0
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm0, %xmm1
; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X64-SSE4-NEXT: paddd %xmm1, %xmm0
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-SSE4-NEXT: paddd %xmm0, %xmm1
-; X64-SSE4-NEXT: movd %xmm1, %eax
+; X64-SSE4-NEXT: paddq %xmm1, %xmm0
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: # kill: def $eax killed $eax killed $rax
; X64-SSE4-NEXT: retq
;
-; AVX1-SLOW-LABEL: test_v16i32_v16i8:
-; AVX1-SLOW: # %bb.0:
-; AVX1-SLOW-NEXT: vbroadcastss {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
-; AVX1-SLOW-NEXT: vandps %ymm2, %ymm0, %ymm0
-; AVX1-SLOW-NEXT: vandps %ymm2, %ymm1, %ymm1
-; AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
-; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; AVX1-SLOW-NEXT: vzeroupper
-; AVX1-SLOW-NEXT: ret{{[l|q]}}
+; X86-AVX1-LABEL: test_v16i32_v16i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vbroadcastss {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; X86-AVX1-NEXT: vandps %ymm2, %ymm1, %ymm1
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vandps %ymm2, %ymm0, %ymm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: vzeroupper
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v16i32_v16i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vbroadcastss {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
+; X64-AVX1-NEXT: vandps %ymm2, %ymm1, %ymm1
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; X64-AVX1-NEXT: vpackusdw %xmm3, %xmm1, %xmm1
+; X64-AVX1-NEXT: vandps %ymm2, %ymm0, %ymm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; X64-AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX1-NEXT: vzeroupper
+; X64-AVX1-NEXT: retq
;
-; AVX1-FAST-LABEL: test_v16i32_v16i8:
-; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vbroadcastss {{.*#+}} ymm2 = [255,255,255,255,255,255,255,255]
-; AVX1-FAST-NEXT: vandps %ymm2, %ymm0, %ymm0
-; AVX1-FAST-NEXT: vandps %ymm2, %ymm1, %ymm1
-; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm3
-; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
-; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; AVX1-FAST-NEXT: vzeroupper
-; AVX1-FAST-NEXT: ret{{[l|q]}}
+; X86-AVX2-LABEL: test_v16i32_v16i8:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; X86-AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: vzeroupper
+; X86-AVX2-NEXT: retl
;
-; AVX2-LABEL: test_v16i32_v16i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: ret{{[l|q]}}
+; X64-AVX2-LABEL: test_v16i32_v16i8:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpbroadcastd {{.*#+}} ymm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; X64-AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX2-NEXT: vzeroupper
+; X64-AVX2-NEXT: retq
;
; AVX512-LABEL: test_v16i32_v16i8:
; AVX512: # %bb.0:
@@ -990,7 +1110,8 @@ define i32 @test_v16i32_v16i8(<16 x i32> %a0) nounwind {
; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%1 = and <16 x i32> %a0, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
@@ -1006,25 +1127,25 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
; X86-SSE2-NEXT: pand %xmm3, %xmm0
; X86-SSE2-NEXT: pand %xmm3, %xmm2
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm4
-; X86-SSE2-NEXT: pand %xmm3, %xmm4
-; X86-SSE2-NEXT: paddd %xmm1, %xmm4
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm5
-; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm1
; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: paddd %xmm5, %xmm1
-; X86-SSE2-NEXT: paddd %xmm4, %xmm1
; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm4
; X86-SSE2-NEXT: pand %xmm3, %xmm4
; X86-SSE2-NEXT: paddd %xmm0, %xmm4
-; X86-SSE2-NEXT: pand 56(%ebp), %xmm3
-; X86-SSE2-NEXT: paddd %xmm2, %xmm3
-; X86-SSE2-NEXT: paddd %xmm4, %xmm3
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm0
+; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: paddd %xmm2, %xmm0
+; X86-SSE2-NEXT: paddd %xmm4, %xmm0
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm2
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: paddd %xmm1, %xmm2
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE2-NEXT: pand %xmm3, %xmm1
+; X86-SSE2-NEXT: pand 8(%ebp), %xmm3
; X86-SSE2-NEXT: paddd %xmm1, %xmm3
+; X86-SSE2-NEXT: paddd %xmm2, %xmm3
+; X86-SSE2-NEXT: paddd %xmm0, %xmm3
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
; X86-SSE2-NEXT: paddd %xmm3, %xmm0
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1037,13 +1158,6 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X64-SSE2-LABEL: test_v32i32_v32i8:
; X64-SSE2: # %bb.0:
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm8 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; X64-SSE2-NEXT: pand %xmm8, %xmm5
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
-; X64-SSE2-NEXT: paddd %xmm5, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm7
-; X64-SSE2-NEXT: pand %xmm8, %xmm3
-; X64-SSE2-NEXT: paddd %xmm7, %xmm3
-; X64-SSE2-NEXT: paddd %xmm1, %xmm3
; X64-SSE2-NEXT: pand %xmm8, %xmm4
; X64-SSE2-NEXT: pand %xmm8, %xmm0
; X64-SSE2-NEXT: paddd %xmm4, %xmm0
@@ -1051,9 +1165,16 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X64-SSE2-NEXT: pand %xmm8, %xmm2
; X64-SSE2-NEXT: paddd %xmm6, %xmm2
; X64-SSE2-NEXT: paddd %xmm0, %xmm2
-; X64-SSE2-NEXT: paddd %xmm3, %xmm2
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; X64-SSE2-NEXT: paddd %xmm2, %xmm0
+; X64-SSE2-NEXT: pand %xmm8, %xmm5
+; X64-SSE2-NEXT: pand %xmm8, %xmm1
+; X64-SSE2-NEXT: paddd %xmm5, %xmm1
+; X64-SSE2-NEXT: pand %xmm8, %xmm7
+; X64-SSE2-NEXT: pand %xmm8, %xmm3
+; X64-SSE2-NEXT: paddd %xmm7, %xmm3
+; X64-SSE2-NEXT: paddd %xmm1, %xmm3
+; X64-SSE2-NEXT: paddd %xmm2, %xmm3
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X64-SSE2-NEXT: paddd %xmm3, %xmm0
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; X64-SSE2-NEXT: paddd %xmm0, %xmm1
; X64-SSE2-NEXT: movd %xmm1, %eax
@@ -1066,25 +1187,25 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm3 = [255,255,255,255]
-; X86-SSE4-NEXT: pand %xmm3, %xmm1
; X86-SSE4-NEXT: pand %xmm3, %xmm0
; X86-SSE4-NEXT: pand %xmm3, %xmm2
-; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm4
-; X86-SSE4-NEXT: pand %xmm3, %xmm4
-; X86-SSE4-NEXT: paddd %xmm1, %xmm4
-; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm5
-; X86-SSE4-NEXT: pand %xmm3, %xmm5
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm1
; X86-SSE4-NEXT: pand %xmm3, %xmm1
-; X86-SSE4-NEXT: paddd %xmm5, %xmm1
-; X86-SSE4-NEXT: paddd %xmm4, %xmm1
; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm4
; X86-SSE4-NEXT: pand %xmm3, %xmm4
; X86-SSE4-NEXT: paddd %xmm0, %xmm4
-; X86-SSE4-NEXT: pand 56(%ebp), %xmm3
-; X86-SSE4-NEXT: paddd %xmm2, %xmm3
-; X86-SSE4-NEXT: paddd %xmm4, %xmm3
+; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm0
+; X86-SSE4-NEXT: pand %xmm3, %xmm0
+; X86-SSE4-NEXT: paddd %xmm2, %xmm0
+; X86-SSE4-NEXT: paddd %xmm4, %xmm0
+; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm2
+; X86-SSE4-NEXT: pand %xmm3, %xmm2
+; X86-SSE4-NEXT: paddd %xmm1, %xmm2
+; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE4-NEXT: pand %xmm3, %xmm1
+; X86-SSE4-NEXT: pand 8(%ebp), %xmm3
; X86-SSE4-NEXT: paddd %xmm1, %xmm3
+; X86-SSE4-NEXT: paddd %xmm2, %xmm3
+; X86-SSE4-NEXT: paddd %xmm0, %xmm3
; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
; X86-SSE4-NEXT: paddd %xmm3, %xmm0
; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1097,13 +1218,6 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X64-SSE4-LABEL: test_v32i32_v32i8:
; X64-SSE4: # %bb.0:
; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm8 = [255,255,255,255]
-; X64-SSE4-NEXT: pand %xmm8, %xmm5
-; X64-SSE4-NEXT: pand %xmm8, %xmm1
-; X64-SSE4-NEXT: paddd %xmm5, %xmm1
-; X64-SSE4-NEXT: pand %xmm8, %xmm7
-; X64-SSE4-NEXT: pand %xmm8, %xmm3
-; X64-SSE4-NEXT: paddd %xmm7, %xmm3
-; X64-SSE4-NEXT: paddd %xmm1, %xmm3
; X64-SSE4-NEXT: pand %xmm8, %xmm4
; X64-SSE4-NEXT: pand %xmm8, %xmm0
; X64-SSE4-NEXT: paddd %xmm4, %xmm0
@@ -1111,9 +1225,16 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X64-SSE4-NEXT: pand %xmm8, %xmm2
; X64-SSE4-NEXT: paddd %xmm6, %xmm2
; X64-SSE4-NEXT: paddd %xmm0, %xmm2
-; X64-SSE4-NEXT: paddd %xmm3, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; X64-SSE4-NEXT: paddd %xmm2, %xmm0
+; X64-SSE4-NEXT: pand %xmm8, %xmm5
+; X64-SSE4-NEXT: pand %xmm8, %xmm1
+; X64-SSE4-NEXT: paddd %xmm5, %xmm1
+; X64-SSE4-NEXT: pand %xmm8, %xmm7
+; X64-SSE4-NEXT: pand %xmm8, %xmm3
+; X64-SSE4-NEXT: paddd %xmm7, %xmm3
+; X64-SSE4-NEXT: paddd %xmm1, %xmm3
+; X64-SSE4-NEXT: paddd %xmm2, %xmm3
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X64-SSE4-NEXT: paddd %xmm3, %xmm0
; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; X64-SSE4-NEXT: paddd %xmm0, %xmm1
; X64-SSE4-NEXT: movd %xmm1, %eax
@@ -1130,17 +1251,17 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X86-AVX1-SLOW-NEXT: vandps %ymm3, %ymm0, %ymm0
; X86-AVX1-SLOW-NEXT: vandps %ymm3, %ymm2, %ymm2
; X86-AVX1-SLOW-NEXT: vandps 8(%ebp), %ymm3, %ymm3
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm4
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm4
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm5
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm6
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm6, %xmm5, %xmm5
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm4, %xmm4
; X86-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm1, %xmm2
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm4, %xmm2
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm3
; X86-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm1, %xmm1
; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm2, %xmm0
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1158,17 +1279,17 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm2, %ymm2
; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm1, %ymm1
; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm3, %ymm3
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm1, %xmm4
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm6, %xmm5
; X64-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm5, %xmm4
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm1, %xmm1
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm4, %xmm0
+; X64-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1188,19 +1309,18 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X86-AVX1-FAST-NEXT: vandps %ymm3, %ymm0, %ymm0
; X86-AVX1-FAST-NEXT: vandps %ymm3, %ymm2, %ymm2
; X86-AVX1-FAST-NEXT: vandps 8(%ebp), %ymm3, %ymm3
-; X86-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm4
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm4
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm5
+; X86-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm6
+; X86-AVX1-FAST-NEXT: vpaddd %xmm6, %xmm5, %xmm5
+; X86-AVX1-FAST-NEXT: vpaddd %xmm5, %xmm4, %xmm4
; X86-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm2
-; X86-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm4, %xmm2
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm3
; X86-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm1
; X86-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpaddd %xmm0, %xmm2, %xmm0
-; X86-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X86-AVX1-FAST-NEXT: movl %ebp, %esp
@@ -1215,19 +1335,18 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm2, %ymm2
; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm1, %ymm1
; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm3, %ymm3
-; X64-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm4
-; X64-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-FAST-NEXT: vpaddd %xmm5, %xmm6, %xmm5
; X64-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm5, %xmm4
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm1
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vpaddd %xmm0, %xmm4, %xmm0
-; X64-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X64-AVX1-FAST-NEXT: vzeroupper
@@ -1281,15 +1400,18 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
;
; AVX512-LABEL: test_v32i32_v32i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpmovdb %zmm0, %xmm0
-; AVX512-NEXT: vpmovdb %zmm1, %xmm1
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; AVX512-NEXT: vpandd %zmm2, %zmm1, %zmm1
+; AVX512-NEXT: vpandd %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
@@ -1306,62 +1428,49 @@ define i16 @test_v2i16_v2i8(<2 x i16> %a0) nounwind {
; X86-SSE-LABEL: test_v2i16_v2i8:
; X86-SSE: # %bb.0:
; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: movdqa %xmm0, %xmm1
-; X86-SSE-NEXT: psrld $16, %xmm1
-; X86-SSE-NEXT: paddw %xmm0, %xmm1
-; X86-SSE-NEXT: movd %xmm1, %eax
+; X86-SSE-NEXT: pxor %xmm1, %xmm1
+; X86-SSE-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE-NEXT: movd %xmm0, %eax
; X86-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE-NEXT: retl
;
; X64-SSE-LABEL: test_v2i16_v2i8:
; X64-SSE: # %bb.0:
; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; X64-SSE-NEXT: movdqa %xmm0, %xmm1
-; X64-SSE-NEXT: psrld $16, %xmm1
-; X64-SSE-NEXT: paddw %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
+; X64-SSE-NEXT: pxor %xmm1, %xmm1
+; X64-SSE-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE-NEXT: movd %xmm0, %eax
; X64-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SSE-NEXT: retq
;
-; X86-AVX1-SLOW-LABEL: test_v2i16_v2i8:
-; X86-AVX1-SLOW: # %bb.0:
-; X86-AVX1-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
-; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-AVX1-SLOW-NEXT: retl
-;
-; X64-AVX1-SLOW-LABEL: test_v2i16_v2i8:
-; X64-AVX1-SLOW: # %bb.0:
-; X64-AVX1-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
-; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-AVX1-SLOW-NEXT: retq
-;
-; X86-AVX1-FAST-LABEL: test_v2i16_v2i8:
-; X86-AVX1-FAST: # %bb.0:
-; X86-AVX1-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-AVX1-FAST-NEXT: retl
+; X86-AVX1-LABEL: test_v2i16_v2i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackuswb {{\.?LCPI[0-9]+_[0-9]+}}+16, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-NEXT: retl
;
-; X64-AVX1-FAST-LABEL: test_v2i16_v2i8:
-; X64-AVX1-FAST: # %bb.0:
-; X64-AVX1-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-AVX1-FAST-NEXT: retq
+; X64-AVX1-LABEL: test_v2i16_v2i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackuswb {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: test_v2i16_v2i8:
; X86-AVX2: # %bb.0:
; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
-; X86-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpackuswb {{\.?LCPI[0-9]+_[0-9]+}}+16, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; X86-AVX2-NEXT: retl
@@ -1369,20 +1478,36 @@ define i16 @test_v2i16_v2i8(<2 x i16> %a0) nounwind {
; X64-AVX2-LABEL: test_v2i16_v2i8:
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
-; X64-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpackuswb {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovd %xmm0, %eax
; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; X64-AVX2-NEXT: retq
;
-; AVX512-LABEL: test_v2i16_v2i8:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: retq
+; AVX512BW-LABEL: test_v2i16_v2i8:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+;
+; AVX512BWVL-LABEL: test_v2i16_v2i8:
+; AVX512BWVL: # %bb.0:
+; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT: vmovd %xmm0, %eax
+; AVX512BWVL-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BWVL-NEXT: vzeroupper
+; AVX512BWVL-NEXT: retq
%1 = and <2 x i16> %a0, <i16 255, i16 255>
%2 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %1)
ret i16 %2
@@ -1461,8 +1586,7 @@ define i16 @test_v4i16_v4i8(<4 x i16> %a0) nounwind {
; X86-AVX1-FAST: # %bb.0:
; X86-AVX1-FAST-NEXT: vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1 # [u,32768,16384,8192,u,u,u,u]
; X86-AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; X86-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X86-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1472,8 +1596,7 @@ define i16 @test_v4i16_v4i8(<4 x i16> %a0) nounwind {
; X64-AVX1-FAST: # %bb.0:
; X64-AVX1-FAST-NEXT: vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 # [u,32768,16384,8192,u,u,u,u]
; X64-AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; X64-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X64-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1536,28 +1659,28 @@ define i16 @test_v8i16_v8i8(<8 x i16> %a0) nounwind {
; X86-SSE-LABEL: test_v8i16_v8i8:
; X86-SSE: # %bb.0:
; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: packuswb %xmm0, %xmm0
; X86-SSE-NEXT: pxor %xmm1, %xmm1
-; X86-SSE-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE-NEXT: movd %xmm1, %eax
+; X86-SSE-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE-NEXT: movd %xmm0, %eax
; X86-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE-NEXT: retl
;
; X64-SSE-LABEL: test_v8i16_v8i8:
; X64-SSE: # %bb.0:
; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; X64-SSE-NEXT: packuswb %xmm0, %xmm0
; X64-SSE-NEXT: pxor %xmm1, %xmm1
-; X64-SSE-NEXT: psadbw %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
+; X64-SSE-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE-NEXT: movd %xmm0, %eax
; X64-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SSE-NEXT: retq
;
; X86-AVX1-LABEL: test_v8i16_v8i8:
; X86-AVX1: # %bb.0:
; X86-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1566,18 +1689,18 @@ define i16 @test_v8i16_v8i8(<8 x i16> %a0) nounwind {
; X64-AVX1-LABEL: test_v8i16_v8i8:
; X64-AVX1: # %bb.0:
; X64-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX1-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $rax
; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: test_v8i16_v8i8:
; X86-AVX2: # %bb.0:
; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1586,22 +1709,34 @@ define i16 @test_v8i16_v8i8(<8 x i16> %a0) nounwind {
; X64-AVX2-LABEL: test_v8i16_v8i8:
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX2-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vmovd %xmm0, %eax
-; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $rax
; X64-AVX2-NEXT: retq
;
-; AVX512-LABEL: test_v8i16_v8i8:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: retq
+; AVX512BW-LABEL: test_v8i16_v8i8:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovq %xmm0, %rax
+; AVX512BW-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+;
+; AVX512BWVL-LABEL: test_v8i16_v8i8:
+; AVX512BWVL: # %bb.0:
+; AVX512BWVL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT: vmovq %xmm0, %rax
+; AVX512BWVL-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX512BWVL-NEXT: vzeroupper
+; AVX512BWVL-NEXT: retq
%1 = and <8 x i16> %a0, <i16 0, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64>
%2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %1)
ret i16 %2
@@ -1634,8 +1769,8 @@ define i16 @test_v16i16_v16i8(<16 x i16> %a0) nounwind {
; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; X64-SSE2-NEXT: paddq %xmm1, %xmm0
-; X64-SSE2-NEXT: movd %xmm0, %eax
-; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $rax
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v16i16_v16i8:
@@ -1664,8 +1799,8 @@ define i16 @test_v16i16_v16i8(<16 x i16> %a0) nounwind {
; X64-SSE4-NEXT: psadbw %xmm0, %xmm1
; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; X64-SSE4-NEXT: paddq %xmm1, %xmm0
-; X64-SSE4-NEXT: movd %xmm0, %eax
-; X64-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: # kill: def $ax killed $ax killed $rax
; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: test_v16i16_v16i8:
@@ -1693,8 +1828,8 @@ define i16 @test_v16i16_v16i8(<16 x i16> %a0) nounwind {
; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $rax
; X64-AVX1-NEXT: vzeroupper
; X64-AVX1-NEXT: retq
;
@@ -1723,8 +1858,8 @@ define i16 @test_v16i16_v16i8(<16 x i16> %a0) nounwind {
; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vmovd %xmm0, %eax
-; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $rax
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
@@ -1737,8 +1872,8 @@ define i16 @test_v16i16_v16i8(<16 x i16> %a0) nounwind {
; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512BW-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vmovd %xmm0, %eax
-; AVX512BW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BW-NEXT: vmovq %xmm0, %rax
+; AVX512BW-NEXT: # kill: def $ax killed $ax killed $rax
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -1750,8 +1885,8 @@ define i16 @test_v16i16_v16i8(<16 x i16> %a0) nounwind {
; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512BWVL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vmovd %xmm0, %eax
-; AVX512BWVL-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BWVL-NEXT: vmovq %xmm0, %rax
+; AVX512BWVL-NEXT: # kill: def $ax killed $ax killed $rax
; AVX512BWVL-NEXT: vzeroupper
; AVX512BWVL-NEXT: retq
%1 = and <16 x i16> %a0, <i16 0, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 0, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64>
@@ -1767,19 +1902,21 @@ define i16 @test_v32i16_v32i8(<32 x i16> %a0) nounwind {
; X86-SSE-NEXT: andl $-16, %esp
; X86-SSE-NEXT: subl $16, %esp
; X86-SSE-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE-NEXT: psrlw $8, %xmm1
+; X86-SSE-NEXT: psrlw $8, %xmm2
; X86-SSE-NEXT: psrlw $8, %xmm0
-; X86-SSE-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE-NEXT: paddw %xmm2, %xmm0
; X86-SSE-NEXT: psrlw $8, %xmm3
-; X86-SSE-NEXT: psrlw $8, %xmm2
-; X86-SSE-NEXT: packuswb %xmm3, %xmm2
-; X86-SSE-NEXT: pxor %xmm1, %xmm1
-; X86-SSE-NEXT: psadbw %xmm1, %xmm2
-; X86-SSE-NEXT: psadbw %xmm1, %xmm0
-; X86-SSE-NEXT: paddq %xmm2, %xmm0
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE-NEXT: paddq %xmm0, %xmm1
-; X86-SSE-NEXT: movd %xmm1, %eax
+; X86-SSE-NEXT: psrlw $8, %xmm1
+; X86-SSE-NEXT: paddw %xmm3, %xmm1
+; X86-SSE-NEXT: paddw %xmm0, %xmm1
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE-NEXT: paddw %xmm1, %xmm0
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE-NEXT: paddw %xmm0, %xmm1
+; X86-SSE-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE-NEXT: psrld $16, %xmm0
+; X86-SSE-NEXT: paddw %xmm1, %xmm0
+; X86-SSE-NEXT: movd %xmm0, %eax
; X86-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE-NEXT: movl %ebp, %esp
; X86-SSE-NEXT: popl %ebp
@@ -1787,55 +1924,78 @@ define i16 @test_v32i16_v32i8(<32 x i16> %a0) nounwind {
;
; X64-SSE-LABEL: test_v32i16_v32i8:
; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: psrlw $8, %xmm1
+; X64-SSE-NEXT: psrlw $8, %xmm2
; X64-SSE-NEXT: psrlw $8, %xmm0
-; X64-SSE-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE-NEXT: paddw %xmm2, %xmm0
; X64-SSE-NEXT: psrlw $8, %xmm3
-; X64-SSE-NEXT: psrlw $8, %xmm2
-; X64-SSE-NEXT: packuswb %xmm3, %xmm2
-; X64-SSE-NEXT: pxor %xmm1, %xmm1
-; X64-SSE-NEXT: psadbw %xmm1, %xmm2
-; X64-SSE-NEXT: psadbw %xmm1, %xmm0
-; X64-SSE-NEXT: paddq %xmm2, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE-NEXT: paddq %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
+; X64-SSE-NEXT: psrlw $8, %xmm1
+; X64-SSE-NEXT: paddw %xmm3, %xmm1
+; X64-SSE-NEXT: paddw %xmm0, %xmm1
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddw %xmm1, %xmm0
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE-NEXT: paddw %xmm0, %xmm1
+; X64-SSE-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE-NEXT: psrld $16, %xmm0
+; X64-SSE-NEXT: paddw %xmm1, %xmm0
+; X64-SSE-NEXT: movd %xmm0, %eax
; X64-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SSE-NEXT: retq
;
-; AVX1-LABEL: test_v32i16_v32i8:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT: vpackuswb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX1-NEXT: vzeroupper
-; AVX1-NEXT: ret{{[l|q]}}
+; AVX1-SLOW-LABEL: test_v32i16_v32i8:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vpsrlw $8, %xmm1, %xmm2
+; AVX1-SLOW-NEXT: vpsrlw $8, %xmm0, %xmm3
+; AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm3, %xmm2
+; AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-SLOW-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-SLOW-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-SLOW-NEXT: vzeroupper
+; AVX1-SLOW-NEXT: ret{{[l|q]}}
+;
+; AVX1-FAST-LABEL: test_v32i16_v32i8:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vpsrlw $8, %xmm1, %xmm2
+; AVX1-FAST-NEXT: vpsrlw $8, %xmm0, %xmm3
+; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm3, %xmm2
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-FAST-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-FAST-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-FAST-NEXT: vzeroupper
+; AVX1-FAST-NEXT: ret{{[l|q]}}
;
; AVX2-LABEL: test_v32i16_v32i8:
; AVX2: # %bb.0:
; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
-; AVX2-NEXT: vpackuswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: vzeroupper
@@ -1844,13 +2004,16 @@ define i16 @test_v32i16_v32i8(<32 x i16> %a0) nounwind {
; AVX512-LABEL: test_v32i16_v32i8:
; AVX512: # %bb.0:
; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512-NEXT: vpmovwb %zmm0, %ymm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: vzeroupper
@@ -1868,30 +2031,32 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [127,127,127,127,127,127,127,127]
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
; X86-SSE2-NEXT: pand %xmm3, %xmm2
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm4
-; X86-SSE2-NEXT: pand %xmm3, %xmm4
-; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm1
; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: packuswb %xmm4, %xmm1
-; X86-SSE2-NEXT: paddb %xmm0, %xmm1
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: packuswb %xmm0, %xmm2
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm0
+; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm4
+; X86-SSE2-NEXT: pand %xmm3, %xmm4
+; X86-SSE2-NEXT: paddw %xmm0, %xmm4
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm0
; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: pand 56(%ebp), %xmm3
-; X86-SSE2-NEXT: packuswb %xmm0, %xmm3
-; X86-SSE2-NEXT: paddb %xmm2, %xmm3
-; X86-SSE2-NEXT: pxor %xmm0, %xmm0
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm3
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE2-NEXT: paddq %xmm3, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm1, %xmm0
+; X86-SSE2-NEXT: paddw %xmm2, %xmm0
+; X86-SSE2-NEXT: paddw %xmm4, %xmm0
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm2
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: paddw %xmm1, %xmm2
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE2-NEXT: pand %xmm3, %xmm1
+; X86-SSE2-NEXT: pand 8(%ebp), %xmm3
+; X86-SSE2-NEXT: paddw %xmm1, %xmm3
+; X86-SSE2-NEXT: paddw %xmm2, %xmm3
+; X86-SSE2-NEXT: paddw %xmm0, %xmm3
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X86-SSE2-NEXT: paddw %xmm3, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT: paddw %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: psrld $16, %xmm0
+; X86-SSE2-NEXT: paddw %xmm1, %xmm0
; X86-SSE2-NEXT: movd %xmm0, %eax
; X86-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE2-NEXT: movl %ebp, %esp
@@ -1901,26 +2066,28 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; X64-SSE2-LABEL: test_v64i16_v64i8:
; X64-SSE2: # %bb.0:
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm8 = [127,127,127,127,127,127,127,127]
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm0
-; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
-; X64-SSE2-NEXT: pand %xmm8, %xmm5
; X64-SSE2-NEXT: pand %xmm8, %xmm4
-; X64-SSE2-NEXT: packuswb %xmm5, %xmm4
-; X64-SSE2-NEXT: paddb %xmm0, %xmm4
-; X64-SSE2-NEXT: pand %xmm8, %xmm3
+; X64-SSE2-NEXT: pand %xmm8, %xmm0
+; X64-SSE2-NEXT: paddw %xmm4, %xmm0
+; X64-SSE2-NEXT: pand %xmm8, %xmm6
; X64-SSE2-NEXT: pand %xmm8, %xmm2
-; X64-SSE2-NEXT: packuswb %xmm3, %xmm2
+; X64-SSE2-NEXT: paddw %xmm6, %xmm2
+; X64-SSE2-NEXT: paddw %xmm0, %xmm2
+; X64-SSE2-NEXT: pand %xmm8, %xmm5
+; X64-SSE2-NEXT: pand %xmm8, %xmm1
+; X64-SSE2-NEXT: paddw %xmm5, %xmm1
; X64-SSE2-NEXT: pand %xmm8, %xmm7
-; X64-SSE2-NEXT: pand %xmm8, %xmm6
-; X64-SSE2-NEXT: packuswb %xmm7, %xmm6
-; X64-SSE2-NEXT: paddb %xmm2, %xmm6
-; X64-SSE2-NEXT: pxor %xmm0, %xmm0
-; X64-SSE2-NEXT: psadbw %xmm0, %xmm6
-; X64-SSE2-NEXT: psadbw %xmm0, %xmm4
-; X64-SSE2-NEXT: paddq %xmm6, %xmm4
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm4, %xmm0
+; X64-SSE2-NEXT: pand %xmm8, %xmm3
+; X64-SSE2-NEXT: paddw %xmm7, %xmm3
+; X64-SSE2-NEXT: paddw %xmm1, %xmm3
+; X64-SSE2-NEXT: paddw %xmm2, %xmm3
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X64-SSE2-NEXT: paddw %xmm3, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE2-NEXT: paddw %xmm0, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT: psrld $16, %xmm0
+; X64-SSE2-NEXT: paddw %xmm1, %xmm0
; X64-SSE2-NEXT: movd %xmm0, %eax
; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SSE2-NEXT: retq
@@ -1932,30 +2099,32 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
; X86-SSE4-NEXT: pmovsxbw {{.*#+}} xmm3 = [127,127,127,127,127,127,127,127]
-; X86-SSE4-NEXT: pand %xmm3, %xmm1
; X86-SSE4-NEXT: pand %xmm3, %xmm0
-; X86-SSE4-NEXT: packuswb %xmm1, %xmm0
; X86-SSE4-NEXT: pand %xmm3, %xmm2
-; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm4
-; X86-SSE4-NEXT: pand %xmm3, %xmm4
-; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm1
; X86-SSE4-NEXT: pand %xmm3, %xmm1
-; X86-SSE4-NEXT: packuswb %xmm4, %xmm1
-; X86-SSE4-NEXT: paddb %xmm0, %xmm1
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm0
-; X86-SSE4-NEXT: pand %xmm3, %xmm0
-; X86-SSE4-NEXT: packuswb %xmm0, %xmm2
-; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm0
+; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm4
+; X86-SSE4-NEXT: pand %xmm3, %xmm4
+; X86-SSE4-NEXT: paddw %xmm0, %xmm4
+; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm0
; X86-SSE4-NEXT: pand %xmm3, %xmm0
-; X86-SSE4-NEXT: pand 56(%ebp), %xmm3
-; X86-SSE4-NEXT: packuswb %xmm0, %xmm3
-; X86-SSE4-NEXT: paddb %xmm2, %xmm3
-; X86-SSE4-NEXT: pxor %xmm0, %xmm0
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm3
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE4-NEXT: paddq %xmm3, %xmm1
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm1, %xmm0
+; X86-SSE4-NEXT: paddw %xmm2, %xmm0
+; X86-SSE4-NEXT: paddw %xmm4, %xmm0
+; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm2
+; X86-SSE4-NEXT: pand %xmm3, %xmm2
+; X86-SSE4-NEXT: paddw %xmm1, %xmm2
+; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE4-NEXT: pand %xmm3, %xmm1
+; X86-SSE4-NEXT: pand 8(%ebp), %xmm3
+; X86-SSE4-NEXT: paddw %xmm1, %xmm3
+; X86-SSE4-NEXT: paddw %xmm2, %xmm3
+; X86-SSE4-NEXT: paddw %xmm0, %xmm3
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X86-SSE4-NEXT: paddw %xmm3, %xmm0
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: paddw %xmm0, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE4-NEXT: psrld $16, %xmm0
+; X86-SSE4-NEXT: paddw %xmm1, %xmm0
; X86-SSE4-NEXT: movd %xmm0, %eax
; X86-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE4-NEXT: movl %ebp, %esp
@@ -1965,91 +2134,153 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; X64-SSE4-LABEL: test_v64i16_v64i8:
; X64-SSE4: # %bb.0:
; X64-SSE4-NEXT: pmovsxbw {{.*#+}} xmm8 = [127,127,127,127,127,127,127,127]
-; X64-SSE4-NEXT: pand %xmm8, %xmm1
-; X64-SSE4-NEXT: pand %xmm8, %xmm0
-; X64-SSE4-NEXT: packuswb %xmm1, %xmm0
-; X64-SSE4-NEXT: pand %xmm8, %xmm5
; X64-SSE4-NEXT: pand %xmm8, %xmm4
-; X64-SSE4-NEXT: packuswb %xmm5, %xmm4
-; X64-SSE4-NEXT: paddb %xmm0, %xmm4
-; X64-SSE4-NEXT: pand %xmm8, %xmm3
+; X64-SSE4-NEXT: pand %xmm8, %xmm0
+; X64-SSE4-NEXT: paddw %xmm4, %xmm0
+; X64-SSE4-NEXT: pand %xmm8, %xmm6
; X64-SSE4-NEXT: pand %xmm8, %xmm2
-; X64-SSE4-NEXT: packuswb %xmm3, %xmm2
+; X64-SSE4-NEXT: paddw %xmm6, %xmm2
+; X64-SSE4-NEXT: paddw %xmm0, %xmm2
+; X64-SSE4-NEXT: pand %xmm8, %xmm5
+; X64-SSE4-NEXT: pand %xmm8, %xmm1
+; X64-SSE4-NEXT: paddw %xmm5, %xmm1
; X64-SSE4-NEXT: pand %xmm8, %xmm7
-; X64-SSE4-NEXT: pand %xmm8, %xmm6
-; X64-SSE4-NEXT: packuswb %xmm7, %xmm6
-; X64-SSE4-NEXT: paddb %xmm2, %xmm6
-; X64-SSE4-NEXT: pxor %xmm0, %xmm0
-; X64-SSE4-NEXT: psadbw %xmm0, %xmm6
-; X64-SSE4-NEXT: psadbw %xmm0, %xmm4
-; X64-SSE4-NEXT: paddq %xmm6, %xmm4
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm4, %xmm0
+; X64-SSE4-NEXT: pand %xmm8, %xmm3
+; X64-SSE4-NEXT: paddw %xmm7, %xmm3
+; X64-SSE4-NEXT: paddw %xmm1, %xmm3
+; X64-SSE4-NEXT: paddw %xmm2, %xmm3
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X64-SSE4-NEXT: paddw %xmm3, %xmm0
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: paddw %xmm0, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE4-NEXT: psrld $16, %xmm0
+; X64-SSE4-NEXT: paddw %xmm1, %xmm0
; X64-SSE4-NEXT: movd %xmm0, %eax
; X64-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SSE4-NEXT: retq
;
-; X86-AVX1-LABEL: test_v64i16_v64i8:
-; X86-AVX1: # %bb.0:
-; X86-AVX1-NEXT: pushl %ebp
-; X86-AVX1-NEXT: movl %esp, %ebp
-; X86-AVX1-NEXT: andl $-32, %esp
-; X86-AVX1-NEXT: subl $32, %esp
-; X86-AVX1-NEXT: vbroadcastss {{.*#+}} ymm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-AVX1-NEXT: vandps %ymm3, %ymm1, %ymm1
-; X86-AVX1-NEXT: vandps %ymm3, %ymm0, %ymm0
-; X86-AVX1-NEXT: vandps %ymm3, %ymm2, %ymm2
-; X86-AVX1-NEXT: vandps 8(%ebp), %ymm3, %ymm3
-; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4
-; X86-AVX1-NEXT: vpackuswb %xmm4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
-; X86-AVX1-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
-; X86-AVX1-NEXT: vpackuswb %xmm4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
-; X86-AVX1-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-AVX1-NEXT: movl %ebp, %esp
-; X86-AVX1-NEXT: popl %ebp
-; X86-AVX1-NEXT: vzeroupper
-; X86-AVX1-NEXT: retl
+; X86-AVX1-SLOW-LABEL: test_v64i16_v64i8:
+; X86-AVX1-SLOW: # %bb.0:
+; X86-AVX1-SLOW-NEXT: pushl %ebp
+; X86-AVX1-SLOW-NEXT: movl %esp, %ebp
+; X86-AVX1-SLOW-NEXT: andl $-32, %esp
+; X86-AVX1-SLOW-NEXT: subl $32, %esp
+; X86-AVX1-SLOW-NEXT: vbroadcastss {{.*#+}} ymm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; X86-AVX1-SLOW-NEXT: vandps %ymm3, %ymm1, %ymm1
+; X86-AVX1-SLOW-NEXT: vandps %ymm3, %ymm0, %ymm0
+; X86-AVX1-SLOW-NEXT: vandps %ymm3, %ymm2, %ymm2
+; X86-AVX1-SLOW-NEXT: vandps 8(%ebp), %ymm3, %ymm3
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm4
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm5
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm6
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm6, %xmm5, %xmm5
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm5, %xmm4, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm1, %xmm1
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-SLOW-NEXT: movl %ebp, %esp
+; X86-AVX1-SLOW-NEXT: popl %ebp
+; X86-AVX1-SLOW-NEXT: vzeroupper
+; X86-AVX1-SLOW-NEXT: retl
;
-; X64-AVX1-LABEL: test_v64i16_v64i8:
-; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vbroadcastss {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-AVX1-NEXT: vandps %ymm4, %ymm0, %ymm0
-; X64-AVX1-NEXT: vandps %ymm4, %ymm2, %ymm2
-; X64-AVX1-NEXT: vandps %ymm4, %ymm1, %ymm1
-; X64-AVX1-NEXT: vandps %ymm4, %ymm3, %ymm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
-; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
-; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4
-; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm2, %xmm2
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
-; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-AVX1-NEXT: vzeroupper
-; X64-AVX1-NEXT: retq
+; X64-AVX1-SLOW-LABEL: test_v64i16_v64i8:
+; X64-AVX1-SLOW: # %bb.0:
+; X64-AVX1-SLOW-NEXT: vbroadcastss {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm0, %ymm0
+; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm2, %ymm2
+; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm1, %ymm1
+; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm3, %ymm3
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm5, %xmm6, %xmm5
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm1, %xmm1
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm0, %xmm0
+; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX1-SLOW-NEXT: vzeroupper
+; X64-AVX1-SLOW-NEXT: retq
+;
+; X86-AVX1-FAST-LABEL: test_v64i16_v64i8:
+; X86-AVX1-FAST: # %bb.0:
+; X86-AVX1-FAST-NEXT: pushl %ebp
+; X86-AVX1-FAST-NEXT: movl %esp, %ebp
+; X86-AVX1-FAST-NEXT: andl $-32, %esp
+; X86-AVX1-FAST-NEXT: subl $32, %esp
+; X86-AVX1-FAST-NEXT: vbroadcastss {{.*#+}} ymm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; X86-AVX1-FAST-NEXT: vandps %ymm3, %ymm1, %ymm1
+; X86-AVX1-FAST-NEXT: vandps %ymm3, %ymm0, %ymm0
+; X86-AVX1-FAST-NEXT: vandps %ymm3, %ymm2, %ymm2
+; X86-AVX1-FAST-NEXT: vandps 8(%ebp), %ymm3, %ymm3
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm4
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm5
+; X86-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm6
+; X86-AVX1-FAST-NEXT: vpaddw %xmm6, %xmm5, %xmm5
+; X86-AVX1-FAST-NEXT: vpaddw %xmm5, %xmm4, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vpaddw %xmm3, %xmm1, %xmm1
+; X86-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-FAST-NEXT: movl %ebp, %esp
+; X86-AVX1-FAST-NEXT: popl %ebp
+; X86-AVX1-FAST-NEXT: vzeroupper
+; X86-AVX1-FAST-NEXT: retl
+;
+; X64-AVX1-FAST-LABEL: test_v64i16_v64i8:
+; X64-AVX1-FAST: # %bb.0:
+; X64-AVX1-FAST-NEXT: vbroadcastss {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm0, %ymm0
+; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm2, %ymm2
+; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm1, %ymm1
+; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm3, %ymm3
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-FAST-NEXT: vpaddw %xmm5, %xmm6, %xmm5
+; X64-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X64-AVX1-FAST-NEXT: vpaddw %xmm3, %xmm1, %xmm1
+; X64-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX1-FAST-NEXT: vzeroupper
+; X64-AVX1-FAST-NEXT: retq
;
; X86-AVX2-LABEL: test_v64i16_v64i8:
; X86-AVX2: # %bb.0:
@@ -2058,20 +2289,21 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; X86-AVX2-NEXT: andl $-32, %esp
; X86-AVX2-NEXT: subl $32, %esp
; X86-AVX2-NEXT: vpbroadcastw {{.*#+}} ymm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
; X86-AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpackuswb %ymm1, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpand 8(%ebp), %ymm3, %ymm1
-; X86-AVX2-NEXT: vpackuswb %ymm1, %ymm2, %ymm1
-; X86-AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpaddw %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpand 8(%ebp), %ymm3, %ymm2
+; X86-AVX2-NEXT: vpaddw %ymm2, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
+; X86-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; X86-AVX2-NEXT: movl %ebp, %esp
@@ -2082,20 +2314,21 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; X64-AVX2-LABEL: test_v64i16_v64i8:
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpbroadcastw {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
-; X64-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpackuswb %ymm1, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm1
; X64-AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2
-; X64-AVX2-NEXT: vpackuswb %ymm1, %ymm2, %ymm1
-; X64-AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpaddw %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm2
+; X64-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpaddw %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
+; X64-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovd %xmm0, %eax
; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; X64-AVX2-NEXT: vzeroupper
@@ -2103,18 +2336,20 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
;
; AVX512-LABEL: test_v64i16_v64i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpmovwb %zmm0, %ymm0
-; AVX512-NEXT: vpmovwb %zmm1, %ymm1
-; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm1
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: vzeroupper
@@ -2151,4 +2386,4 @@ declare i8 @llvm.vector.reduce.add.v64i8(<64 x i8>)
declare i8 @llvm.vector.reduce.add.v128i8(<128 x i8>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; AVX: {{.*}}
-; SSE2: {{.*}}
+; SSE: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll b/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll
index 3b041304e252d..813114552505c 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll
@@ -379,7 +379,7 @@ define i64 @test_v8i64_v8i8(<8 x i8> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovsxbq %xmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -403,7 +403,7 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
; X86-SSE2-NEXT: pcmpgtd %xmm2, %xmm1
; X86-SSE2-NEXT: movdqu %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-SSE2-NEXT: movdqa %xmm2, %xmm4
-; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3]
; X86-SSE2-NEXT: psrad $24, %xmm7
@@ -411,7 +411,7 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
; X86-SSE2-NEXT: pcmpgtd %xmm7, %xmm5
; X86-SSE2-NEXT: movdqu %xmm5, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-SSE2-NEXT: movdqa %xmm7, %xmm1
-; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
; X86-SSE2-NEXT: paddq %xmm4, %xmm1
; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
; X86-SSE2-NEXT: psrad $24, %xmm3
@@ -419,23 +419,23 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
; X86-SSE2-NEXT: pcmpgtd %xmm3, %xmm4
; X86-SSE2-NEXT: movdqu %xmm4, (%esp) # 16-byte Spill
; X86-SSE2-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
; X86-SSE2-NEXT: psrad $24, %xmm4
; X86-SSE2-NEXT: pxor %xmm5, %xmm5
; X86-SSE2-NEXT: pcmpgtd %xmm4, %xmm5
; X86-SSE2-NEXT: movdqa %xmm4, %xmm0
-; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
; X86-SSE2-NEXT: paddq %xmm6, %xmm0
; X86-SSE2-NEXT: paddq %xmm1, %xmm0
; X86-SSE2-NEXT: movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; X86-SSE2-NEXT: movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm1[2],xmm7[3],xmm1[3]
; X86-SSE2-NEXT: paddq %xmm2, %xmm7
; X86-SSE2-NEXT: movdqu (%esp), %xmm1 # 16-byte Reload
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
; X86-SSE2-NEXT: paddq %xmm3, %xmm4
; X86-SSE2-NEXT: paddq %xmm7, %xmm4
; X86-SSE2-NEXT: paddq %xmm0, %xmm4
@@ -456,33 +456,33 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
; X64-SSE2-NEXT: pxor %xmm3, %xmm3
; X64-SSE2-NEXT: pcmpgtd %xmm2, %xmm3
; X64-SSE2-NEXT: movdqa %xmm2, %xmm5
-; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
; X64-SSE2-NEXT: psrad $24, %xmm0
; X64-SSE2-NEXT: pxor %xmm7, %xmm7
; X64-SSE2-NEXT: pcmpgtd %xmm0, %xmm7
; X64-SSE2-NEXT: movdqa %xmm0, %xmm8
-; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
; X64-SSE2-NEXT: paddq %xmm5, %xmm8
; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
; X64-SSE2-NEXT: psrad $24, %xmm4
; X64-SSE2-NEXT: pxor %xmm5, %xmm5
; X64-SSE2-NEXT: pcmpgtd %xmm4, %xmm5
; X64-SSE2-NEXT: movdqa %xmm4, %xmm9
-; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm5[2],xmm9[3],xmm5[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1]
; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]
; X64-SSE2-NEXT: psrad $24, %xmm6
; X64-SSE2-NEXT: pcmpgtd %xmm6, %xmm1
; X64-SSE2-NEXT: movdqa %xmm6, %xmm10
-; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm1[2],xmm10[3],xmm1[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm1[0],xmm10[1],xmm1[1]
; X64-SSE2-NEXT: paddq %xmm9, %xmm10
; X64-SSE2-NEXT: paddq %xmm8, %xmm10
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm7[2],xmm0[3],xmm7[3]
; X64-SSE2-NEXT: paddq %xmm2, %xmm0
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm1[2],xmm6[3],xmm1[3]
; X64-SSE2-NEXT: paddq %xmm4, %xmm6
; X64-SSE2-NEXT: paddq %xmm0, %xmm6
; X64-SSE2-NEXT: paddq %xmm10, %xmm6
@@ -493,95 +493,95 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
;
; X86-SSE4-LABEL: test_v16i64_v16i8:
; X86-SSE4: # %bb.0:
-; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE4-NEXT: pmovsxbq %xmm0, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: pmovsxbq %xmm2, %xmm2
+; X86-SSE4-NEXT: paddq %xmm1, %xmm2
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; X86-SSE4-NEXT: pmovsxbq %xmm1, %xmm3
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: pmovsxbq %xmm1, %xmm1
+; X86-SSE4-NEXT: paddq %xmm3, %xmm1
+; X86-SSE4-NEXT: paddq %xmm2, %xmm1
; X86-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE4-NEXT: psrldq {{.*#+}} xmm2 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: pmovsxbq %xmm2, %xmm2
; X86-SSE4-NEXT: movdqa %xmm0, %xmm3
-; X86-SSE4-NEXT: pmovsxbq %xmm0, %xmm4
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm0[3,3,3,3]
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,1,1]
-; X86-SSE4-NEXT: psrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X86-SSE4-NEXT: pmovsxbq %xmm0, %xmm0
-; X86-SSE4-NEXT: psrld $16, %xmm1
-; X86-SSE4-NEXT: pmovsxbq %xmm1, %xmm1
-; X86-SSE4-NEXT: paddq %xmm0, %xmm1
-; X86-SSE4-NEXT: psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X86-SSE4-NEXT: pmovsxbq %xmm2, %xmm0
-; X86-SSE4-NEXT: psrlq $48, %xmm3
-; X86-SSE4-NEXT: pmovsxbq %xmm3, %xmm2
-; X86-SSE4-NEXT: paddq %xmm0, %xmm2
-; X86-SSE4-NEXT: paddq %xmm1, %xmm2
-; X86-SSE4-NEXT: pmovsxbq %xmm5, %xmm0
-; X86-SSE4-NEXT: paddq %xmm4, %xmm0
-; X86-SSE4-NEXT: pmovsxbq %xmm6, %xmm1
-; X86-SSE4-NEXT: pmovsxbq %xmm7, %xmm3
-; X86-SSE4-NEXT: paddq %xmm1, %xmm3
-; X86-SSE4-NEXT: paddq %xmm0, %xmm3
+; X86-SSE4-NEXT: psrld $16, %xmm3
+; X86-SSE4-NEXT: pmovsxbq %xmm3, %xmm3
; X86-SSE4-NEXT: paddq %xmm2, %xmm3
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE4-NEXT: psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: pmovsxbq %xmm2, %xmm2
+; X86-SSE4-NEXT: psrlq $48, %xmm0
+; X86-SSE4-NEXT: pmovsxbq %xmm0, %xmm0
+; X86-SSE4-NEXT: paddq %xmm2, %xmm0
; X86-SSE4-NEXT: paddq %xmm3, %xmm0
-; X86-SSE4-NEXT: movd %xmm0, %eax
-; X86-SSE4-NEXT: pextrd $1, %xmm0, %edx
+; X86-SSE4-NEXT: paddq %xmm1, %xmm0
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm0, %xmm1
+; X86-SSE4-NEXT: movd %xmm1, %eax
+; X86-SSE4-NEXT: pextrd $1, %xmm1, %edx
; X86-SSE4-NEXT: retl
;
; X64-SSE4-LABEL: test_v16i64_v16i8:
; X64-SSE4: # %bb.0:
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE4-NEXT: pmovsxbq %xmm0, %xmm4
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm0[3,3,3,3]
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,1,1]
-; X64-SSE4-NEXT: psrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-SSE4-NEXT: pmovsxbq %xmm0, %xmm0
-; X64-SSE4-NEXT: psrld $16, %xmm1
-; X64-SSE4-NEXT: pmovsxbq %xmm1, %xmm1
-; X64-SSE4-NEXT: paddq %xmm0, %xmm1
-; X64-SSE4-NEXT: psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-SSE4-NEXT: pmovsxbq %xmm2, %xmm0
-; X64-SSE4-NEXT: psrlq $48, %xmm3
-; X64-SSE4-NEXT: pmovsxbq %xmm3, %xmm2
-; X64-SSE4-NEXT: paddq %xmm0, %xmm2
+; X64-SSE4-NEXT: pmovsxbq %xmm0, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: pmovsxbq %xmm2, %xmm2
; X64-SSE4-NEXT: paddq %xmm1, %xmm2
-; X64-SSE4-NEXT: pmovsxbq %xmm5, %xmm0
-; X64-SSE4-NEXT: paddq %xmm4, %xmm0
-; X64-SSE4-NEXT: pmovsxbq %xmm6, %xmm1
-; X64-SSE4-NEXT: pmovsxbq %xmm7, %xmm3
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; X64-SSE4-NEXT: pmovsxbq %xmm1, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: pmovsxbq %xmm3, %xmm3
; X64-SSE4-NEXT: paddq %xmm1, %xmm3
-; X64-SSE4-NEXT: paddq %xmm0, %xmm3
; X64-SSE4-NEXT: paddq %xmm2, %xmm3
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE4-NEXT: psrldq {{.*#+}} xmm1 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: pmovsxbq %xmm1, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X64-SSE4-NEXT: psrld $16, %xmm2
+; X64-SSE4-NEXT: pmovsxbq %xmm2, %xmm2
+; X64-SSE4-NEXT: paddq %xmm1, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE4-NEXT: psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: pmovsxbq %xmm1, %xmm1
+; X64-SSE4-NEXT: psrlq $48, %xmm0
+; X64-SSE4-NEXT: pmovsxbq %xmm0, %xmm0
+; X64-SSE4-NEXT: paddq %xmm1, %xmm0
+; X64-SSE4-NEXT: paddq %xmm2, %xmm0
; X64-SSE4-NEXT: paddq %xmm3, %xmm0
-; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm0, %xmm1
+; X64-SSE4-NEXT: movq %xmm1, %rax
; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: test_v16i64_v16i8:
; X86-AVX1: # %bb.0:
-; X86-AVX1-NEXT: vpsrld $16, %xmm0, %xmm1
-; X86-AVX1-NEXT: vpmovsxbq %xmm1, %xmm2
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-NEXT: vpmovsxbw %xmm1, %xmm3
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
-; X86-AVX1-NEXT: vpmovsxwq %xmm4, %xmm4
-; X86-AVX1-NEXT: vpaddq %xmm4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpmovsxbw %xmm0, %xmm4
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
+; X86-AVX1-NEXT: vpmovsxbw %xmm0, %xmm1
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X86-AVX1-NEXT: vpmovsxwq %xmm2, %xmm2
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpmovsxbw %xmm3, %xmm4
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
; X86-AVX1-NEXT: vpmovsxwq %xmm5, %xmm5
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[3,3,3,3]
-; X86-AVX1-NEXT: vpmovsxwq %xmm6, %xmm6
-; X86-AVX1-NEXT: vpaddq %xmm6, %xmm5, %xmm5
; X86-AVX1-NEXT: vpaddq %xmm5, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
-; X86-AVX1-NEXT: vpmovsxwq %xmm4, %xmm4
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
-; X86-AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm4, %xmm3
+; X86-AVX1-NEXT: vpmovsxbq %xmm0, %xmm5
+; X86-AVX1-NEXT: vpmovsxbq %xmm3, %xmm3
+; X86-AVX1-NEXT: vpaddq %xmm3, %xmm5, %xmm3
+; X86-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
+; X86-AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
; X86-AVX1-NEXT: vpmovsxbq %xmm0, %xmm0
-; X86-AVX1-NEXT: vpmovsxbq %xmm1, %xmm1
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
+; X86-AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
; X86-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; X86-AVX1-NEXT: vpmovsxwq %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[3,3,3,3]
+; X86-AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
+; X86-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
@@ -590,30 +590,30 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
;
; X64-AVX1-LABEL: test_v16i64_v16i8:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vpsrld $16, %xmm0, %xmm1
-; X64-AVX1-NEXT: vpmovsxbq %xmm1, %xmm1
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpmovsxbw %xmm2, %xmm3
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
-; X64-AVX1-NEXT: vpmovsxwq %xmm4, %xmm4
-; X64-AVX1-NEXT: vpaddq %xmm4, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpmovsxbw %xmm0, %xmm4
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
+; X64-AVX1-NEXT: vpmovsxbw %xmm0, %xmm1
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X64-AVX1-NEXT: vpmovsxwq %xmm2, %xmm2
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpmovsxbw %xmm3, %xmm4
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
; X64-AVX1-NEXT: vpmovsxwq %xmm5, %xmm5
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[3,3,3,3]
-; X64-AVX1-NEXT: vpmovsxwq %xmm6, %xmm6
-; X64-AVX1-NEXT: vpaddq %xmm6, %xmm5, %xmm5
-; X64-AVX1-NEXT: vpaddq %xmm5, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
-; X64-AVX1-NEXT: vpmovsxwq %xmm4, %xmm4
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
-; X64-AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
-; X64-AVX1-NEXT: vpaddq %xmm3, %xmm4, %xmm3
+; X64-AVX1-NEXT: vpaddq %xmm5, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpmovsxbq %xmm0, %xmm5
+; X64-AVX1-NEXT: vpmovsxbq %xmm3, %xmm3
+; X64-AVX1-NEXT: vpaddq %xmm3, %xmm5, %xmm3
+; X64-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
+; X64-AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
; X64-AVX1-NEXT: vpmovsxbq %xmm0, %xmm0
-; X64-AVX1-NEXT: vpmovsxbq %xmm2, %xmm2
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
+; X64-AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
; X64-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; X64-AVX1-NEXT: vpmovsxwq %xmm1, %xmm1
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[3,3,3,3]
+; X64-AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
+; X64-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
@@ -670,7 +670,7 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
; AVX512-NEXT: vpmovsxwq %xmm0, %zmm0
; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -852,7 +852,7 @@ define i32 @test_v8i32_v8i8(<8 x i8> %a0) nounwind {
; AVX1-FAST-NEXT: vpmovsxbd %xmm0, %xmm1
; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; AVX1-FAST-NEXT: vpmovsxbd %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
@@ -961,8 +961,7 @@ define i32 @test_v16i32_v16i8(<16 x i8> %a0) nounwind {
; AVX1-FAST-NEXT: vpmovsxbd %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm1, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: ret{{[l|q]}}
@@ -987,7 +986,7 @@ define i32 @test_v16i32_v16i8(<16 x i8> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovsxbd %xmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1006,28 +1005,28 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
; X86-SSE2-LABEL: test_v32i32_v32i8:
; X86-SSE2: # %bb.0:
; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
; X86-SSE2-NEXT: psrad $24, %xmm4
; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
; X86-SSE2-NEXT: psrad $24, %xmm5
; X86-SSE2-NEXT: paddd %xmm4, %xmm5
; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm1[4],xmm6[5],xmm1[5],xmm6[6],xmm1[6],xmm6[7],xmm1[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3]
; X86-SSE2-NEXT: psrad $24, %xmm6
; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
-; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
; X86-SSE2-NEXT: psrad $24, %xmm0
; X86-SSE2-NEXT: paddd %xmm6, %xmm0
; X86-SSE2-NEXT: paddd %xmm5, %xmm0
-; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
; X86-SSE2-NEXT: psrad $24, %xmm2
-; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
; X86-SSE2-NEXT: psrad $24, %xmm3
; X86-SSE2-NEXT: paddd %xmm2, %xmm3
-; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
; X86-SSE2-NEXT: psrad $24, %xmm1
-; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
; X86-SSE2-NEXT: psrad $24, %xmm2
; X86-SSE2-NEXT: paddd %xmm1, %xmm2
; X86-SSE2-NEXT: paddd %xmm3, %xmm2
@@ -1042,28 +1041,28 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
; X64-SSE2-LABEL: test_v32i32_v32i8:
; X64-SSE2: # %bb.0:
; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
; X64-SSE2-NEXT: psrad $24, %xmm3
; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; X64-SSE2-NEXT: psrad $24, %xmm5
; X64-SSE2-NEXT: paddd %xmm3, %xmm5
; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; X64-SSE2-NEXT: psrad $24, %xmm3
; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
; X64-SSE2-NEXT: psrad $24, %xmm6
; X64-SSE2-NEXT: paddd %xmm3, %xmm6
; X64-SSE2-NEXT: paddd %xmm5, %xmm6
-; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
; X64-SSE2-NEXT: psrad $24, %xmm2
-; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
; X64-SSE2-NEXT: psrad $24, %xmm3
; X64-SSE2-NEXT: paddd %xmm2, %xmm3
-; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
; X64-SSE2-NEXT: psrad $24, %xmm1
-; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
; X64-SSE2-NEXT: psrad $24, %xmm0
; X64-SSE2-NEXT: paddd %xmm1, %xmm0
; X64-SSE2-NEXT: paddd %xmm3, %xmm0
@@ -1077,23 +1076,23 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
;
; X86-SSE4-LABEL: test_v32i32_v32i8:
; X86-SSE4: # %bb.0:
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; X86-SSE4-NEXT: pmovsxbd %xmm2, %xmm2
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; X86-SSE4-NEXT: pmovsxbd %xmm3, %xmm3
; X86-SSE4-NEXT: paddd %xmm2, %xmm3
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X86-SSE4-NEXT: pmovsxbd %xmm2, %xmm4
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; X86-SSE4-NEXT: pmovsxbd %xmm2, %xmm2
+; X86-SSE4-NEXT: pmovsxbd %xmm1, %xmm4
+; X86-SSE4-NEXT: pmovsxbd %xmm0, %xmm2
; X86-SSE4-NEXT: paddd %xmm4, %xmm2
; X86-SSE4-NEXT: paddd %xmm3, %xmm2
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
; X86-SSE4-NEXT: pmovsxbd %xmm3, %xmm3
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
; X86-SSE4-NEXT: pmovsxbd %xmm4, %xmm4
; X86-SSE4-NEXT: paddd %xmm3, %xmm4
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
; X86-SSE4-NEXT: pmovsxbd %xmm1, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; X86-SSE4-NEXT: pmovsxbd %xmm0, %xmm0
; X86-SSE4-NEXT: paddd %xmm1, %xmm0
; X86-SSE4-NEXT: paddd %xmm4, %xmm0
@@ -1107,23 +1106,23 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
;
; X64-SSE4-LABEL: test_v32i32_v32i8:
; X64-SSE4: # %bb.0:
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; X64-SSE4-NEXT: pmovsxbd %xmm2, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; X64-SSE4-NEXT: pmovsxbd %xmm3, %xmm3
; X64-SSE4-NEXT: paddd %xmm2, %xmm3
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X64-SSE4-NEXT: pmovsxbd %xmm2, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
-; X64-SSE4-NEXT: pmovsxbd %xmm4, %xmm4
+; X64-SSE4-NEXT: pmovsxbd %xmm1, %xmm2
+; X64-SSE4-NEXT: pmovsxbd %xmm0, %xmm4
; X64-SSE4-NEXT: paddd %xmm2, %xmm4
; X64-SSE4-NEXT: paddd %xmm3, %xmm4
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
; X64-SSE4-NEXT: pmovsxbd %xmm2, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
; X64-SSE4-NEXT: pmovsxbd %xmm3, %xmm3
; X64-SSE4-NEXT: paddd %xmm2, %xmm3
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
; X64-SSE4-NEXT: pmovsxbd %xmm1, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; X64-SSE4-NEXT: pmovsxbd %xmm0, %xmm0
; X64-SSE4-NEXT: paddd %xmm1, %xmm0
; X64-SSE4-NEXT: paddd %xmm3, %xmm0
@@ -1138,27 +1137,27 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
; AVX1-SLOW-LABEL: test_v32i32_v32i8:
; AVX1-SLOW: # %bb.0:
; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; AVX1-SLOW-NEXT: vpmovsxbd %xmm2, %xmm2
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; AVX1-SLOW-NEXT: vpmovsxbd %xmm3, %xmm3
; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[3,3,3,3]
-; AVX1-SLOW-NEXT: vpmovsxbd %xmm3, %xmm3
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
-; AVX1-SLOW-NEXT: vpmovsxbd %xmm4, %xmm4
+; AVX1-SLOW-NEXT: vpmovsxbd %xmm1, %xmm3
+; AVX1-SLOW-NEXT: vpmovsxbd %xmm0, %xmm4
; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm4, %xmm3
-; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm2, %xmm2
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
; AVX1-SLOW-NEXT: vpmovsxbd %xmm3, %xmm3
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
; AVX1-SLOW-NEXT: vpmovsxbd %xmm4, %xmm4
; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
; AVX1-SLOW-NEXT: vpmovsxbd %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX1-SLOW-NEXT: vpmovsxbd %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm2, %xmm0
; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1170,29 +1169,28 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
; AVX1-FAST-LABEL: test_v32i32_v32i8:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; AVX1-FAST-NEXT: vpmovsxbd %xmm2, %xmm2
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; AVX1-FAST-NEXT: vpmovsxbd %xmm3, %xmm3
; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[3,3,3,3]
-; AVX1-FAST-NEXT: vpmovsxbd %xmm3, %xmm3
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
-; AVX1-FAST-NEXT: vpmovsxbd %xmm4, %xmm4
+; AVX1-FAST-NEXT: vpmovsxbd %xmm1, %xmm3
+; AVX1-FAST-NEXT: vpmovsxbd %xmm0, %xmm4
; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm4, %xmm3
-; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm2, %xmm2
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
; AVX1-FAST-NEXT: vpmovsxbd %xmm3, %xmm3
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
; AVX1-FAST-NEXT: vpmovsxbd %xmm4, %xmm4
; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
; AVX1-FAST-NEXT: vpmovsxbd %xmm1, %xmm1
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX1-FAST-NEXT: vpmovsxbd %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm2, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: vzeroupper
@@ -1227,7 +1225,7 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
; AVX512-NEXT: vpmovsxbd %xmm0, %zmm0
; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1347,8 +1345,7 @@ define i16 @test_v4i16_v4i8(<4 x i8> %a0) nounwind {
; AVX1-FAST-LABEL: test_v4i16_v4i8:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vpmovsxbw %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1521,7 +1518,7 @@ define i16 @test_v16i16_v16i8(<16 x i8> %a0) nounwind {
; AVX1-FAST-NEXT: vpmovsxbw %xmm0, %xmm1
; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; AVX1-FAST-NEXT: vpmovsxbw %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm0, %xmm1, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
@@ -1647,10 +1644,8 @@ define i16 @test_v32i16_v32i8(<32 x i8> %a0) nounwind {
; AVX1-FAST-NEXT: vpmovsxbw %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1680,7 +1675,7 @@ define i16 @test_v32i16_v32i8(<32 x i8> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovsxbw %ymm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1706,25 +1701,25 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
; X86-SSE2-NEXT: psraw $8, %xmm4
-; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
; X86-SSE2-NEXT: psraw $8, %xmm5
; X86-SSE2-NEXT: paddw %xmm4, %xmm5
-; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm3[8],xmm6[9],xmm3[9],xmm6[10],xmm3[10],xmm6[11],xmm3[11],xmm6[12],xmm3[12],xmm6[13],xmm3[13],xmm6[14],xmm3[14],xmm6[15],xmm3[15]
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3],xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
; X86-SSE2-NEXT: psraw $8, %xmm6
-; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
; X86-SSE2-NEXT: psraw $8, %xmm4
; X86-SSE2-NEXT: paddw %xmm6, %xmm4
; X86-SSE2-NEXT: paddw %xmm5, %xmm4
-; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X86-SSE2-NEXT: psraw $8, %xmm2
-; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X86-SSE2-NEXT: psraw $8, %xmm0
; X86-SSE2-NEXT: paddw %xmm2, %xmm0
-; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
; X86-SSE2-NEXT: psraw $8, %xmm2
-; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X86-SSE2-NEXT: psraw $8, %xmm1
; X86-SSE2-NEXT: paddw %xmm2, %xmm1
; X86-SSE2-NEXT: paddw %xmm0, %xmm1
@@ -1744,25 +1739,25 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
;
; X64-SSE2-LABEL: test_v64i16_v64i8:
; X64-SSE2: # %bb.0:
-; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
; X64-SSE2-NEXT: psraw $8, %xmm4
-; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
; X64-SSE2-NEXT: psraw $8, %xmm5
; X64-SSE2-NEXT: paddw %xmm4, %xmm5
-; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
; X64-SSE2-NEXT: psraw $8, %xmm4
-; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3],xmm6[4],xmm1[4],xmm6[5],xmm1[5],xmm6[6],xmm1[6],xmm6[7],xmm1[7]
; X64-SSE2-NEXT: psraw $8, %xmm6
; X64-SSE2-NEXT: paddw %xmm4, %xmm6
; X64-SSE2-NEXT: paddw %xmm5, %xmm6
-; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X64-SSE2-NEXT: psraw $8, %xmm2
-; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X64-SSE2-NEXT: psraw $8, %xmm0
; X64-SSE2-NEXT: paddw %xmm2, %xmm0
-; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
; X64-SSE2-NEXT: psraw $8, %xmm2
-; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X64-SSE2-NEXT: psraw $8, %xmm1
; X64-SSE2-NEXT: paddw %xmm2, %xmm1
; X64-SSE2-NEXT: paddw %xmm0, %xmm1
@@ -1784,26 +1779,26 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
; X86-SSE4-NEXT: movl %esp, %ebp
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm4
-; X86-SSE4-NEXT: pmovsxbw %xmm2, %xmm3
-; X86-SSE4-NEXT: pmovsxbw %xmm0, %xmm5
-; X86-SSE4-NEXT: paddw %xmm3, %xmm5
-; X86-SSE4-NEXT: pmovsxbw %xmm4, %xmm6
-; X86-SSE4-NEXT: pmovsxbw %xmm1, %xmm3
-; X86-SSE4-NEXT: paddw %xmm6, %xmm3
-; X86-SSE4-NEXT: paddw %xmm5, %xmm3
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm3
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; X86-SSE4-NEXT: pmovsxbw %xmm4, %xmm4
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: pmovsxbw %xmm5, %xmm5
+; X86-SSE4-NEXT: paddw %xmm4, %xmm5
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; X86-SSE4-NEXT: pmovsxbw %xmm4, %xmm4
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: pmovsxbw %xmm6, %xmm6
+; X86-SSE4-NEXT: paddw %xmm4, %xmm6
+; X86-SSE4-NEXT: paddw %xmm5, %xmm6
; X86-SSE4-NEXT: pmovsxbw %xmm2, %xmm2
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; X86-SSE4-NEXT: pmovsxbw %xmm0, %xmm0
; X86-SSE4-NEXT: paddw %xmm2, %xmm0
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
-; X86-SSE4-NEXT: pmovsxbw %xmm2, %xmm2
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: pmovsxbw %xmm3, %xmm2
; X86-SSE4-NEXT: pmovsxbw %xmm1, %xmm1
; X86-SSE4-NEXT: paddw %xmm2, %xmm1
; X86-SSE4-NEXT: paddw %xmm0, %xmm1
-; X86-SSE4-NEXT: paddw %xmm3, %xmm1
+; X86-SSE4-NEXT: paddw %xmm6, %xmm1
; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; X86-SSE4-NEXT: paddw %xmm1, %xmm0
; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1819,21 +1814,21 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
;
; X64-SSE4-LABEL: test_v64i16_v64i8:
; X64-SSE4: # %bb.0:
-; X64-SSE4-NEXT: pmovsxbw %xmm2, %xmm4
-; X64-SSE4-NEXT: pmovsxbw %xmm0, %xmm5
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; X64-SSE4-NEXT: pmovsxbw %xmm4, %xmm4
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: pmovsxbw %xmm5, %xmm5
; X64-SSE4-NEXT: paddw %xmm4, %xmm5
-; X64-SSE4-NEXT: pmovsxbw %xmm3, %xmm4
-; X64-SSE4-NEXT: pmovsxbw %xmm1, %xmm6
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; X64-SSE4-NEXT: pmovsxbw %xmm4, %xmm4
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: pmovsxbw %xmm6, %xmm6
; X64-SSE4-NEXT: paddw %xmm4, %xmm6
; X64-SSE4-NEXT: paddw %xmm5, %xmm6
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
; X64-SSE4-NEXT: pmovsxbw %xmm2, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; X64-SSE4-NEXT: pmovsxbw %xmm0, %xmm0
; X64-SSE4-NEXT: paddw %xmm2, %xmm0
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; X64-SSE4-NEXT: pmovsxbw %xmm2, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: pmovsxbw %xmm3, %xmm2
; X64-SSE4-NEXT: pmovsxbw %xmm1, %xmm1
; X64-SSE4-NEXT: paddw %xmm2, %xmm1
; X64-SSE4-NEXT: paddw %xmm0, %xmm1
@@ -1851,27 +1846,27 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
;
; AVX1-SLOW-LABEL: test_v64i16_v64i8:
; AVX1-SLOW: # %bb.0:
-; AVX1-SLOW-NEXT: vpmovsxbw %xmm1, %xmm2
-; AVX1-SLOW-NEXT: vpmovsxbw %xmm0, %xmm3
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovsxbw %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovsxbw %xmm3, %xmm3
; AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm3, %xmm2
; AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm3
-; AVX1-SLOW-NEXT: vpmovsxbw %xmm3, %xmm4
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovsxbw %xmm4, %xmm4
; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm5
-; AVX1-SLOW-NEXT: vpmovsxbw %xmm5, %xmm6
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovsxbw %xmm6, %xmm6
; AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm6, %xmm4
; AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm2, %xmm2
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX1-SLOW-NEXT: vpmovsxbw %xmm1, %xmm1
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; AVX1-SLOW-NEXT: vpmovsxbw %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
-; AVX1-SLOW-NEXT: vpmovsxbw %xmm1, %xmm1
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[2,3,2,3]
-; AVX1-SLOW-NEXT: vpmovsxbw %xmm3, %xmm3
+; AVX1-SLOW-NEXT: vpmovsxbw %xmm3, %xmm1
+; AVX1-SLOW-NEXT: vpmovsxbw %xmm5, %xmm3
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm3, %xmm1
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1885,31 +1880,29 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
;
; AVX1-FAST-LABEL: test_v64i16_v64i8:
; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vpmovsxbw %xmm1, %xmm2
-; AVX1-FAST-NEXT: vpmovsxbw %xmm0, %xmm3
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovsxbw %xmm2, %xmm2
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovsxbw %xmm3, %xmm3
; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm3, %xmm2
; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm3
-; AVX1-FAST-NEXT: vpmovsxbw %xmm3, %xmm4
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovsxbw %xmm4, %xmm4
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm5
-; AVX1-FAST-NEXT: vpmovsxbw %xmm5, %xmm6
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovsxbw %xmm6, %xmm6
; AVX1-FAST-NEXT: vpaddw %xmm4, %xmm6, %xmm4
; AVX1-FAST-NEXT: vpaddw %xmm4, %xmm2, %xmm2
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX1-FAST-NEXT: vpmovsxbw %xmm1, %xmm1
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; AVX1-FAST-NEXT: vpmovsxbw %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
-; AVX1-FAST-NEXT: vpmovsxbw %xmm1, %xmm1
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[2,3,2,3]
-; AVX1-FAST-NEXT: vpmovsxbw %xmm3, %xmm3
+; AVX1-FAST-NEXT: vpmovsxbw %xmm3, %xmm1
+; AVX1-FAST-NEXT: vpmovsxbw %xmm5, %xmm3
; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm3, %xmm1
; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpaddw %xmm0, %xmm2, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1947,7 +1940,7 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
; AVX512-NEXT: vpmovsxbw %ymm0, %zmm0
; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll b/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
index 9d13de751812f..6d32535872ced 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
@@ -219,61 +219,155 @@ define i64 @test_v8i64_v8i8(<8 x i8> %a0) nounwind {
; X86-SSE2-LABEL: test_v8i64_v8i8:
; X86-SSE2: # %bb.0:
; X86-SSE2-NEXT: pxor %xmm1, %xmm1
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-SSE2-NEXT: packuswb %xmm3, %xmm2
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE2-NEXT: packuswb %xmm3, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
; X86-SSE2-NEXT: xorl %edx, %edx
; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v8i64_v8i8:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pxor %xmm1, %xmm1
-; X64-SSE-NEXT: psadbw %xmm0, %xmm1
-; X64-SSE-NEXT: movq %xmm1, %rax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v8i64_v8i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm2
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X64-SSE2-NEXT: packuswb %xmm3, %xmm2
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-SSE2-NEXT: packuswb %xmm3, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm2, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v8i64_v8i8:
; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; X86-SSE4-NEXT: pxor %xmm1, %xmm1
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE4-NEXT: movd %xmm1, %eax
-; X86-SSE4-NEXT: pextrd $1, %xmm1, %edx
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: pextrd $1, %xmm0, %edx
; X86-SSE4-NEXT: retl
;
+; X64-SSE4-LABEL: test_v8i64_v8i8:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: retq
+;
; X86-AVX1-LABEL: test_v8i64_v8i8:
; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT: vpsrld $16, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; X86-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; X86-AVX1-NEXT: vbroadcastsd {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0]
+; X86-AVX1-NEXT: vandps %ymm2, %ymm0, %ymm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vandps %ymm2, %ymm1, %ymm1
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
+; X86-AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: vpextrd $1, %xmm0, %edx
+; X86-AVX1-NEXT: vzeroupper
; X86-AVX1-NEXT: retl
;
; X64-AVX1-LABEL: test_v8i64_v8i8:
; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX1-NEXT: vpsrld $16, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm2, %ymm1
+; X64-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
+; X64-AVX1-NEXT: vbroadcastsd {{.*#+}} ymm2 = [255,255,255,255]
+; X64-AVX1-NEXT: vandps %ymm2, %ymm0, %ymm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; X64-AVX1-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
+; X64-AVX1-NEXT: vandps %ymm2, %ymm1, %ymm1
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
+; X64-AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: vzeroupper
; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: test_v8i64_v8i8:
; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: vpextrd $1, %xmm0, %edx
+; X86-AVX2-NEXT: vzeroupper
; X86-AVX2-NEXT: retl
;
; X64-AVX2-LABEL: test_v8i64_v8i8:
; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
; AVX512-LABEL: test_v8i64_v8i8:
; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovzxbq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,xmm0[4],zero,zero,zero,zero,zero,zero,zero,xmm0[5],zero,zero,zero,zero,zero,zero,zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vpmovqb %zmm0, %xmm0
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%1 = zext <8 x i8> %a0 to <8 x i64>
%2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %1)
@@ -424,42 +518,73 @@ define i32 @test_v4i32(<4 x i8> %a0) nounwind {
; SSE2-LABEL: test_v4i32:
; SSE2: # %bb.0:
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT: packuswb %xmm1, %xmm0
+; SSE2-NEXT: packuswb %xmm1, %xmm0
; SSE2-NEXT: psadbw %xmm1, %xmm0
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: ret{{[l|q]}}
;
; SSE4-LABEL: test_v4i32:
; SSE4: # %bb.0:
+; SSE4-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
; SSE4-NEXT: pxor %xmm1, %xmm1
-; SSE4-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; SSE4-NEXT: packusdw %xmm1, %xmm0
+; SSE4-NEXT: packuswb %xmm1, %xmm0
; SSE4-NEXT: psadbw %xmm1, %xmm0
; SSE4-NEXT: movd %xmm0, %eax
; SSE4-NEXT: ret{{[l|q]}}
;
-; AVX1-LABEL: test_v4i32:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
-; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: ret{{[l|q]}}
+; X86-AVX1-LABEL: test_v4i32:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: vzeroupper
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v4i32:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-NEXT: vzeroupper
+; X64-AVX1-NEXT: retq
;
; AVX2-LABEL: test_v4i32:
; AVX2: # %bb.0:
+; AVX2-NEXT: # kill: def $xmm0 killed $xmm0 def $ymm0
+; AVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[0],zero,zero,zero,ymm0[1],zero,zero,zero,ymm0[2],zero,zero,zero,ymm0[3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: ret{{[l|q]}}
;
; AVX512-LABEL: test_v4i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%1 = zext <4 x i8> %a0 to <4 x i32>
%2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)
@@ -467,92 +592,385 @@ define i32 @test_v4i32(<4 x i8> %a0) nounwind {
}
define i32 @test_v8i32_v8i8(<8 x i8> %a0) nounwind {
-; SSE-LABEL: test_v8i32_v8i8:
-; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm1, %xmm1
-; SSE-NEXT: psadbw %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX-LABEL: test_v8i32_v8i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: ret{{[l|q]}}
+; SSE2-LABEL: test_v8i32_v8i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT: packuswb %xmm1, %xmm0
+; SSE2-NEXT: psadbw %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: ret{{[l|q]}}
+;
+; SSE4-LABEL: test_v8i32_v8i8:
+; SSE4: # %bb.0:
+; SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE4-NEXT: pxor %xmm1, %xmm1
+; SSE4-NEXT: packuswb %xmm1, %xmm0
+; SSE4-NEXT: psadbw %xmm1, %xmm0
+; SSE4-NEXT: movd %xmm0, %eax
+; SSE4-NEXT: ret{{[l|q]}}
+;
+; X86-AVX1-LABEL: test_v8i32_v8i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: vzeroupper
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v8i32_v8i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X64-AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X64-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-NEXT: vzeroupper
+; X64-AVX1-NEXT: retq
+;
+; AVX2-LABEL: test_v8i32_v8i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: ret{{[l|q]}}
+;
+; AVX512-LABEL: test_v8i32_v8i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%1 = zext <8 x i8> %a0 to <8 x i32>
%2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
ret i32 %2
}
define i32 @test_v16i32_v16i8(<16 x i8> %a0) nounwind {
-; SSE-LABEL: test_v16i32_v16i8:
-; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm1, %xmm1
-; SSE-NEXT: psadbw %xmm0, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; SSE-NEXT: paddq %xmm1, %xmm0
-; SSE-NEXT: movd %xmm0, %eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX-LABEL: test_v16i32_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: ret{{[l|q]}}
+; X86-SSE-LABEL: test_v16i32_v16i8:
+; X86-SSE: # %bb.0:
+; X86-SSE-NEXT: pxor %xmm1, %xmm1
+; X86-SSE-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE-NEXT: paddq %xmm1, %xmm0
+; X86-SSE-NEXT: movd %xmm0, %eax
+; X86-SSE-NEXT: retl
+;
+; X64-SSE-LABEL: test_v16i32_v16i8:
+; X64-SSE: # %bb.0:
+; X64-SSE-NEXT: pxor %xmm1, %xmm1
+; X64-SSE-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddq %xmm1, %xmm0
+; X64-SSE-NEXT: movq %xmm0, %rax
+; X64-SSE-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-SSE-NEXT: retq
+;
+; X86-AVX1-LABEL: test_v16i32_v16i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v16i32_v16i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX1-NEXT: retq
+;
+; X86-AVX2-LABEL: test_v16i32_v16i8:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: test_v16i32_v16i8:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX2-NEXT: retq
+;
+; AVX512-LABEL: test_v16i32_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX512-NEXT: retq
%1 = zext <16 x i8> %a0 to <16 x i32>
%2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)
ret i32 %2
}
define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
-; SSE-LABEL: test_v32i32_v32i8:
-; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm2, %xmm2
-; SSE-NEXT: psadbw %xmm2, %xmm1
-; SSE-NEXT: psadbw %xmm2, %xmm0
-; SSE-NEXT: paddq %xmm1, %xmm0
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE-NEXT: paddq %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX1-LABEL: test_v32i32_v32i8:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: vzeroupper
-; AVX1-NEXT: ret{{[l|q]}}
+; X86-SSE2-LABEL: test_v32i32_v32i8:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pxor %xmm2, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm5
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm6
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; X86-SSE2-NEXT: paddd %xmm5, %xmm6
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm7
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X86-SSE2-NEXT: paddd %xmm7, %xmm5
+; X86-SSE2-NEXT: paddd %xmm6, %xmm5
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X86-SSE2-NEXT: paddd %xmm4, %xmm3
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; X86-SSE2-NEXT: paddd %xmm1, %xmm0
+; X86-SSE2-NEXT: paddd %xmm3, %xmm0
+; X86-SSE2-NEXT: paddd %xmm5, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE2-NEXT: paddd %xmm0, %xmm1
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X86-SSE2-NEXT: paddd %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: retl
+;
+; X64-SSE2-LABEL: test_v32i32_v32i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pxor %xmm2, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm4
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm5
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm6
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; X64-SSE2-NEXT: paddd %xmm4, %xmm6
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm4
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm7
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; X64-SSE2-NEXT: paddd %xmm4, %xmm7
+; X64-SSE2-NEXT: paddd %xmm6, %xmm7
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; X64-SSE2-NEXT: paddd %xmm3, %xmm5
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; X64-SSE2-NEXT: paddd %xmm1, %xmm0
+; X64-SSE2-NEXT: paddd %xmm5, %xmm0
+; X64-SSE2-NEXT: paddd %xmm7, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE2-NEXT: paddd %xmm0, %xmm1
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X64-SSE2-NEXT: paddd %xmm1, %xmm0
+; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: test_v32i32_v32i8:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; X86-SSE4-NEXT: paddd %xmm2, %xmm3
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-SSE4-NEXT: paddd %xmm4, %xmm2
+; X86-SSE4-NEXT: paddd %xmm3, %xmm2
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; X86-SSE4-NEXT: paddd %xmm3, %xmm4
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-SSE4-NEXT: paddd %xmm1, %xmm0
+; X86-SSE4-NEXT: paddd %xmm4, %xmm0
+; X86-SSE4-NEXT: paddd %xmm2, %xmm0
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: paddd %xmm0, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X86-SSE4-NEXT: paddd %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: test_v32i32_v32i8:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; X64-SSE4-NEXT: paddd %xmm2, %xmm3
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-SSE4-NEXT: paddd %xmm2, %xmm4
+; X64-SSE4-NEXT: paddd %xmm3, %xmm4
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; X64-SSE4-NEXT: paddd %xmm2, %xmm3
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-SSE4-NEXT: paddd %xmm1, %xmm0
+; X64-SSE4-NEXT: paddd %xmm3, %xmm0
+; X64-SSE4-NEXT: paddd %xmm4, %xmm0
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: paddd %xmm0, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X64-SSE4-NEXT: paddd %xmm1, %xmm0
+; X64-SSE4-NEXT: movd %xmm0, %eax
+; X64-SSE4-NEXT: retq
+;
+; AVX1-SLOW-LABEL: test_v32i32_v32i8:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm2, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT: vzeroupper
+; AVX1-SLOW-NEXT: ret{{[l|q]}}
+;
+; AVX1-FAST-LABEL: test_v32i32_v32i8:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm2, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; AVX1-FAST-NEXT: vzeroupper
+; AVX1-FAST-NEXT: ret{{[l|q]}}
;
; AVX2-LABEL: test_v32i32_v32i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: ret{{[l|q]}}
;
; AVX512-LABEL: test_v32i32_v32i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
@@ -566,83 +984,384 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
;
define i16 @test_v2i16_v2i8(<2 x i8> %a0) nounwind {
-; SSE2-LABEL: test_v2i16_v2i8:
+; X86-SSE2-LABEL: test_v2i16_v2i8:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT: retl
+;
+; X64-SSE2-LABEL: test_v2i16_v2i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT: retq
+;
+; SSE4-LABEL: test_v2i16_v2i8:
+; SSE4: # %bb.0:
+; SSE4-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE4-NEXT: pxor %xmm1, %xmm1
+; SSE4-NEXT: packuswb %xmm1, %xmm0
+; SSE4-NEXT: psadbw %xmm1, %xmm0
+; SSE4-NEXT: movd %xmm0, %eax
+; SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT: ret{{[l|q]}}
+;
+; X86-AVX1-LABEL: test_v2i16_v2i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT: vpackuswb {{\.?LCPI[0-9]+_[0-9]+}}+16, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v2i16_v2i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X64-AVX1-NEXT: vpackuswb {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX1-NEXT: retq
+;
+; AVX2-LABEL: test_v2i16_v2i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: ret{{[l|q]}}
+;
+; AVX512BW-LABEL: test_v2i16_v2i8:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+;
+; AVX512BWVL-LABEL: test_v2i16_v2i8:
+; AVX512BWVL: # %bb.0:
+; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT: vmovd %xmm0, %eax
+; AVX512BWVL-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BWVL-NEXT: vzeroupper
+; AVX512BWVL-NEXT: retq
+ %1 = zext <2 x i8> %a0 to <2 x i16>
+ %2 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %1)
+ ret i16 %2
+}
+
+define i16 @test_v4i16_v4i8(<4 x i8> %a0) nounwind {
+; X86-SSE2-LABEL: test_v4i16_v4i8:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT: retl
+;
+; X64-SSE2-LABEL: test_v4i16_v4i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT: retq
+;
+; SSE4-LABEL: test_v4i16_v4i8:
+; SSE4: # %bb.0:
+; SSE4-NEXT: pshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,zero,zero
+; SSE4-NEXT: pxor %xmm1, %xmm1
+; SSE4-NEXT: packuswb %xmm1, %xmm0
+; SSE4-NEXT: psadbw %xmm1, %xmm0
+; SSE4-NEXT: movd %xmm0, %eax
+; SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT: ret{{[l|q]}}
+;
+; X86-AVX1-LABEL: test_v4i16_v4i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT: vpackuswb {{\.?LCPI[0-9]+_[0-9]+}}+16, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v4i16_v4i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X64-AVX1-NEXT: vpackuswb {{\.?LCPI[0-9]+_[0-9]+}}+16(%rip), %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX1-NEXT: retq
+;
+; AVX2-LABEL: test_v4i16_v4i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: ret{{[l|q]}}
+;
+; AVX512BW-LABEL: test_v4i16_v4i8:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+;
+; AVX512BWVL-LABEL: test_v4i16_v4i8:
+; AVX512BWVL: # %bb.0:
+; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BWVL-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT: vmovd %xmm0, %eax
+; AVX512BWVL-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BWVL-NEXT: vzeroupper
+; AVX512BWVL-NEXT: retq
+ %1 = zext <4 x i8> %a0 to <4 x i16>
+ %2 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %1)
+ ret i16 %2
+
+}
+
+define i16 @test_v8i16_v8i8(<8 x i8> %a0) nounwind {
+; SSE2-LABEL: test_v8i16_v8i8:
; SSE2: # %bb.0:
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $16, %xmm1
-; SSE2-NEXT: paddw %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: packuswb %xmm1, %xmm0
+; SSE2-NEXT: psadbw %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: ret{{[l|q]}}
;
-; SSE4-LABEL: test_v2i16_v2i8:
+; SSE4-LABEL: test_v8i16_v8i8:
; SSE4: # %bb.0:
; SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE4-NEXT: movdqa %xmm0, %xmm1
-; SSE4-NEXT: psrld $16, %xmm1
-; SSE4-NEXT: paddw %xmm0, %xmm1
-; SSE4-NEXT: movd %xmm1, %eax
+; SSE4-NEXT: pxor %xmm1, %xmm1
+; SSE4-NEXT: packuswb %xmm1, %xmm0
+; SSE4-NEXT: psadbw %xmm1, %xmm0
+; SSE4-NEXT: movd %xmm0, %eax
; SSE4-NEXT: # kill: def $ax killed $ax killed $eax
; SSE4-NEXT: ret{{[l|q]}}
;
-; AVX1-SLOW-LABEL: test_v2i16_v2i8:
-; AVX1-SLOW: # %bb.0:
-; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX1-SLOW-NEXT: ret{{[l|q]}}
+; X86-AVX1-LABEL: test_v8i16_v8i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-NEXT: vzeroupper
+; X86-AVX1-NEXT: retl
;
-; AVX1-FAST-LABEL: test_v2i16_v2i8:
-; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX1-FAST-NEXT: ret{{[l|q]}}
+; X64-AVX1-LABEL: test_v8i16_v8i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX1-NEXT: vzeroupper
+; X64-AVX1-NEXT: retq
;
-; AVX2-LABEL: test_v2i16_v2i8:
+; AVX2-LABEL: test_v8i16_v8i8:
; AVX2: # %bb.0:
; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: ret{{[l|q]}}
;
-; AVX512-LABEL: test_v2i16_v2i8:
+; AVX512BW-LABEL: test_v8i16_v8i8:
+; AVX512BW: # %bb.0:
+; AVX512BW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+;
+; AVX512BWVL-LABEL: test_v8i16_v8i8:
+; AVX512BWVL: # %bb.0:
+; AVX512BWVL-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
+; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT: vmovd %xmm0, %eax
+; AVX512BWVL-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BWVL-NEXT: vzeroupper
+; AVX512BWVL-NEXT: retq
+ %1 = zext <8 x i8> %a0 to <8 x i16>
+ %2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %1)
+ ret i16 %2
+}
+
+define i16 @test_v16i16_v16i8(<16 x i8> %a0) nounwind {
+; X86-SSE-LABEL: test_v16i16_v16i8:
+; X86-SSE: # %bb.0:
+; X86-SSE-NEXT: pxor %xmm1, %xmm1
+; X86-SSE-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE-NEXT: paddq %xmm1, %xmm0
+; X86-SSE-NEXT: movd %xmm0, %eax
+; X86-SSE-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE-NEXT: retl
+;
+; X64-SSE-LABEL: test_v16i16_v16i8:
+; X64-SSE: # %bb.0:
+; X64-SSE-NEXT: pxor %xmm1, %xmm1
+; X64-SSE-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddq %xmm1, %xmm0
+; X64-SSE-NEXT: movq %xmm0, %rax
+; X64-SSE-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-SSE-NEXT: retq
+;
+; X86-AVX1-LABEL: test_v16i16_v16i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v16i16_v16i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX1-NEXT: retq
+;
+; X86-AVX2-LABEL: test_v16i16_v16i8:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: test_v16i16_v16i8:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX2-NEXT: retq
+;
+; AVX512-LABEL: test_v16i16_v16i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $rax
; AVX512-NEXT: retq
- %1 = zext <2 x i8> %a0 to <2 x i16>
- %2 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %1)
+ %1 = zext <16 x i8> %a0 to <16 x i16>
+ %2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %1)
ret i16 %2
}
-define i16 @test_v4i16_v4i8(<4 x i8> %a0) nounwind {
-; SSE2-LABEL: test_v4i16_v4i8:
+define i16 @test_v32i16_v32i8(<32 x i8> %a0) nounwind {
+; SSE2-LABEL: test_v32i16_v32i8:
; SSE2: # %bb.0:
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT: paddw %xmm3, %xmm4
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; SSE2-NEXT: paddw %xmm1, %xmm0
+; SSE2-NEXT: paddw %xmm4, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddw %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; SSE2-NEXT: paddw %xmm1, %xmm0
-; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: psrld $16, %xmm1
+; SSE2-NEXT: paddw %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: ret{{[l|q]}}
;
-; SSE4-LABEL: test_v4i16_v4i8:
+; SSE4-LABEL: test_v32i16_v32i8:
; SSE4: # %bb.0:
-; SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE4-NEXT: pxor %xmm2, %xmm2
+; SSE4-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE4-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; SSE4-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE4-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; SSE4-NEXT: paddw %xmm1, %xmm0
+; SSE4-NEXT: paddw %xmm3, %xmm4
+; SSE4-NEXT: paddw %xmm0, %xmm4
+; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
+; SSE4-NEXT: paddw %xmm4, %xmm0
; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; SSE4-NEXT: paddw %xmm0, %xmm1
; SSE4-NEXT: movdqa %xmm1, %xmm0
@@ -652,148 +1371,79 @@ define i16 @test_v4i16_v4i8(<4 x i8> %a0) nounwind {
; SSE4-NEXT: # kill: def $ax killed $ax killed $eax
; SSE4-NEXT: ret{{[l|q]}}
;
-; AVX1-SLOW-LABEL: test_v4i16_v4i8:
+; AVX1-SLOW-LABEL: test_v32i16_v32i8:
; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; AVX1-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
; AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-SLOW-NEXT: vzeroupper
; AVX1-SLOW-NEXT: ret{{[l|q]}}
;
-; AVX1-FAST-LABEL: test_v4i16_v4i8:
+; AVX1-FAST-LABEL: test_v32i16_v32i8:
; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-FAST-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-FAST-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; AVX1-FAST-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX1-FAST-NEXT: vpaddw %xmm3, %xmm2, %xmm2
+; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-FAST-NEXT: vzeroupper
; AVX1-FAST-NEXT: ret{{[l|q]}}
;
-; AVX2-LABEL: test_v4i16_v4i8:
+; AVX2-LABEL: test_v32i16_v32i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: ret{{[l|q]}}
-;
-; AVX512-LABEL: test_v4i16_v4i8:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: retq
- %1 = zext <4 x i8> %a0 to <4 x i16>
- %2 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %1)
- ret i16 %2
-
-}
-
-define i16 @test_v8i16_v8i8(<8 x i8> %a0) nounwind {
-; SSE-LABEL: test_v8i16_v8i8:
-; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm1, %xmm1
-; SSE-NEXT: psadbw %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX-LABEL: test_v8i16_v8i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: ret{{[l|q]}}
- %1 = zext <8 x i8> %a0 to <8 x i16>
- %2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %1)
- ret i16 %2
-}
-
-define i16 @test_v16i16_v16i8(<16 x i8> %a0) nounwind {
-; SSE-LABEL: test_v16i16_v16i8:
-; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm1, %xmm1
-; SSE-NEXT: psadbw %xmm0, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; SSE-NEXT: paddq %xmm1, %xmm0
-; SSE-NEXT: movd %xmm0, %eax
-; SSE-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX-LABEL: test_v16i16_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: ret{{[l|q]}}
- %1 = zext <16 x i8> %a0 to <16 x i16>
- %2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %1)
- ret i16 %2
-}
-
-define i16 @test_v32i16_v32i8(<32 x i8> %a0) nounwind {
-; SSE-LABEL: test_v32i16_v32i8:
-; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm2, %xmm2
-; SSE-NEXT: psadbw %xmm2, %xmm1
-; SSE-NEXT: psadbw %xmm2, %xmm0
-; SSE-NEXT: paddq %xmm1, %xmm0
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE-NEXT: paddq %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX1-LABEL: test_v32i16_v32i8:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX1-NEXT: vzeroupper
-; AVX1-NEXT: ret{{[l|q]}}
-;
-; AVX2-LABEL: test_v32i16_v32i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: ret{{[l|q]}}
;
; AVX512-LABEL: test_v32i16_v32i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: vzeroupper
@@ -804,73 +1454,222 @@ define i16 @test_v32i16_v32i8(<32 x i8> %a0) nounwind {
}
define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
-; X86-SSE-LABEL: test_v64i16_v64i8:
-; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: pushl %ebp
-; X86-SSE-NEXT: movl %esp, %ebp
-; X86-SSE-NEXT: andl $-16, %esp
-; X86-SSE-NEXT: subl $16, %esp
-; X86-SSE-NEXT: pxor %xmm3, %xmm3
-; X86-SSE-NEXT: psadbw %xmm3, %xmm2
-; X86-SSE-NEXT: psadbw %xmm3, %xmm0
-; X86-SSE-NEXT: paddq %xmm2, %xmm0
-; X86-SSE-NEXT: psadbw %xmm3, %xmm1
-; X86-SSE-NEXT: psadbw 8(%ebp), %xmm3
-; X86-SSE-NEXT: paddq %xmm1, %xmm3
-; X86-SSE-NEXT: paddq %xmm0, %xmm3
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
-; X86-SSE-NEXT: paddq %xmm3, %xmm0
-; X86-SSE-NEXT: movd %xmm0, %eax
-; X86-SSE-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-SSE-NEXT: movl %ebp, %esp
-; X86-SSE-NEXT: popl %ebp
-; X86-SSE-NEXT: retl
+; X86-SSE2-LABEL: test_v64i16_v64i8:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pushl %ebp
+; X86-SSE2-NEXT: movl %esp, %ebp
+; X86-SSE2-NEXT: andl $-16, %esp
+; X86-SSE2-NEXT: subl $16, %esp
+; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT: pxor %xmm4, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm5
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm6
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; X86-SSE2-NEXT: paddw %xmm5, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm7
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3],xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7]
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm5
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X86-SSE2-NEXT: paddw %xmm7, %xmm5
+; X86-SSE2-NEXT: paddw %xmm6, %xmm5
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
+; X86-SSE2-NEXT: paddw %xmm2, %xmm0
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; X86-SSE2-NEXT: paddw %xmm3, %xmm1
+; X86-SSE2-NEXT: paddw %xmm0, %xmm1
+; X86-SSE2-NEXT: paddw %xmm5, %xmm1
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE2-NEXT: paddw %xmm1, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT: paddw %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: psrld $16, %xmm0
+; X86-SSE2-NEXT: paddw %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT: movl %ebp, %esp
+; X86-SSE2-NEXT: popl %ebp
+; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v64i16_v64i8:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pxor %xmm4, %xmm4
-; X64-SSE-NEXT: psadbw %xmm4, %xmm3
-; X64-SSE-NEXT: psadbw %xmm4, %xmm1
-; X64-SSE-NEXT: paddq %xmm3, %xmm1
-; X64-SSE-NEXT: psadbw %xmm4, %xmm2
-; X64-SSE-NEXT: psadbw %xmm4, %xmm0
-; X64-SSE-NEXT: paddq %xmm2, %xmm0
-; X64-SSE-NEXT: paddq %xmm1, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE-NEXT: paddq %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
-; X64-SSE-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v64i16_v64i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pxor %xmm4, %xmm4
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm5
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm6
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; X64-SSE2-NEXT: paddw %xmm5, %xmm6
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm5
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm7
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3],xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7]
+; X64-SSE2-NEXT: paddw %xmm5, %xmm7
+; X64-SSE2-NEXT: paddw %xmm6, %xmm7
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
+; X64-SSE2-NEXT: paddw %xmm2, %xmm0
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; X64-SSE2-NEXT: paddw %xmm3, %xmm1
+; X64-SSE2-NEXT: paddw %xmm0, %xmm1
+; X64-SSE2-NEXT: paddw %xmm7, %xmm1
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE2-NEXT: paddw %xmm1, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE2-NEXT: paddw %xmm0, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT: psrld $16, %xmm0
+; X64-SSE2-NEXT: paddw %xmm1, %xmm0
+; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: test_v64i16_v64i8:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pushl %ebp
+; X86-SSE4-NEXT: movl %esp, %ebp
+; X86-SSE4-NEXT: andl $-16, %esp
+; X86-SSE4-NEXT: subl $16, %esp
+; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm5
+; X86-SSE4-NEXT: pxor %xmm6, %xmm6
+; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; X86-SSE4-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15]
+; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-SSE4-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm6[8],xmm0[9],xmm6[9],xmm0[10],xmm6[10],xmm0[11],xmm6[11],xmm0[12],xmm6[12],xmm0[13],xmm6[13],xmm0[14],xmm6[14],xmm0[15],xmm6[15]
+; X86-SSE4-NEXT: paddw %xmm2, %xmm0
+; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; X86-SSE4-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; X86-SSE4-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
+; X86-SSE4-NEXT: paddw %xmm5, %xmm1
+; X86-SSE4-NEXT: paddw %xmm0, %xmm1
+; X86-SSE4-NEXT: paddw %xmm4, %xmm3
+; X86-SSE4-NEXT: paddw %xmm7, %xmm2
+; X86-SSE4-NEXT: paddw %xmm3, %xmm2
+; X86-SSE4-NEXT: paddw %xmm1, %xmm2
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; X86-SSE4-NEXT: paddw %xmm2, %xmm0
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: paddw %xmm0, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE4-NEXT: psrld $16, %xmm0
+; X86-SSE4-NEXT: paddw %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE4-NEXT: movl %ebp, %esp
+; X86-SSE4-NEXT: popl %ebp
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: test_v64i16_v64i8:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pxor %xmm4, %xmm4
+; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; X64-SSE4-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15]
+; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-SSE4-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
+; X64-SSE4-NEXT: paddw %xmm2, %xmm0
+; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; X64-SSE4-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
+; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm7 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; X64-SSE4-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; X64-SSE4-NEXT: paddw %xmm3, %xmm1
+; X64-SSE4-NEXT: paddw %xmm0, %xmm1
+; X64-SSE4-NEXT: paddw %xmm5, %xmm6
+; X64-SSE4-NEXT: paddw %xmm2, %xmm7
+; X64-SSE4-NEXT: paddw %xmm6, %xmm7
+; X64-SSE4-NEXT: paddw %xmm1, %xmm7
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm7[2,3,2,3]
+; X64-SSE4-NEXT: paddw %xmm7, %xmm0
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: paddw %xmm0, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE4-NEXT: psrld $16, %xmm0
+; X64-SSE4-NEXT: paddw %xmm1, %xmm0
+; X64-SSE4-NEXT: movd %xmm0, %eax
+; X64-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE4-NEXT: retq
+;
+; AVX1-SLOW-LABEL: test_v64i16_v64i8:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; AVX1-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm4, %xmm3
+; AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm4
+; AVX1-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
+; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm6
+; AVX1-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15]
+; AVX1-SLOW-NEXT: vpaddw %xmm5, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm3, %xmm2
+; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm3, %xmm1
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-SLOW-NEXT: vzeroupper
+; AVX1-SLOW-NEXT: ret{{[l|q]}}
;
-; AVX1-LABEL: test_v64i16_v64i8:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT: vpsadbw %xmm3, %xmm4, %xmm4
-; AVX1-NEXT: vpaddq %xmm2, %xmm4, %xmm2
-; AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX1-NEXT: vzeroupper
-; AVX1-NEXT: ret{{[l|q]}}
+; AVX1-FAST-LABEL: test_v64i16_v64i8:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-FAST-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; AVX1-FAST-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX1-FAST-NEXT: vpaddw %xmm3, %xmm4, %xmm3
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm4
+; AVX1-FAST-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm6
+; AVX1-FAST-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15]
+; AVX1-FAST-NEXT: vpaddw %xmm5, %xmm2, %xmm2
+; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm3, %xmm2
+; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm3, %xmm1
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-FAST-NEXT: vzeroupper
+; AVX1-FAST-NEXT: ret{{[l|q]}}
;
; AVX2-LABEL: test_v64i16_v64i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpsadbw %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpsadbw %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT: vpaddw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm0, %ymm2, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: vzeroupper
@@ -878,14 +1677,20 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
;
; AVX512-LABEL: test_v64i16_v64i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: vzeroupper
@@ -922,8 +1727,9 @@ declare i8 @llvm.vector.reduce.add.v32i8(<32 x i8>)
declare i8 @llvm.vector.reduce.add.v64i8(<64 x i8>)
declare i8 @llvm.vector.reduce.add.v128i8(<128 x i8>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; AVX512BW: {{.*}}
-; AVX512BWVL: {{.*}}
+; AVX: {{.*}}
+; AVX1: {{.*}}
+; SSE: {{.*}}
; SSE41: {{.*}}
; X64-AVX1-FAST: {{.*}}
; X64-AVX1-SLOW: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add.ll b/llvm/test/CodeGen/X86/vector-reduce-add.ll
index ff5de0e587505..0c754d3738a25 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add.ll
@@ -265,7 +265,7 @@ define i64 @test_v8i64(<8 x i64> %a0) nounwind {
; AVX512-LABEL: test_v8i64:
; AVX512: # %bb.0:
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -285,17 +285,17 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE2-NEXT: paddq 56(%ebp), %xmm2
-; X86-SSE2-NEXT: paddq 24(%ebp), %xmm0
-; X86-SSE2-NEXT: paddq %xmm2, %xmm0
; X86-SSE2-NEXT: paddq 72(%ebp), %xmm3
; X86-SSE2-NEXT: paddq 40(%ebp), %xmm1
; X86-SSE2-NEXT: paddq %xmm3, %xmm1
-; X86-SSE2-NEXT: paddq %xmm0, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE2-NEXT: paddq 56(%ebp), %xmm2
+; X86-SSE2-NEXT: paddq 24(%ebp), %xmm0
+; X86-SSE2-NEXT: paddq %xmm2, %xmm0
; X86-SSE2-NEXT: paddq %xmm1, %xmm0
-; X86-SSE2-NEXT: movd %xmm0, %eax
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE2-NEXT: paddq %xmm0, %xmm1
+; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; X86-SSE2-NEXT: movd %xmm0, %edx
; X86-SSE2-NEXT: movl %ebp, %esp
; X86-SSE2-NEXT: popl %ebp
@@ -303,16 +303,16 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
;
; X64-SSE-LABEL: test_v16i64:
; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: paddq %xmm6, %xmm2
-; X64-SSE-NEXT: paddq %xmm4, %xmm0
-; X64-SSE-NEXT: paddq %xmm2, %xmm0
; X64-SSE-NEXT: paddq %xmm7, %xmm3
; X64-SSE-NEXT: paddq %xmm5, %xmm1
; X64-SSE-NEXT: paddq %xmm3, %xmm1
-; X64-SSE-NEXT: paddq %xmm0, %xmm1
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddq %xmm6, %xmm2
+; X64-SSE-NEXT: paddq %xmm4, %xmm0
+; X64-SSE-NEXT: paddq %xmm2, %xmm0
; X64-SSE-NEXT: paddq %xmm1, %xmm0
-; X64-SSE-NEXT: movq %xmm0, %rax
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE-NEXT: paddq %xmm0, %xmm1
+; X64-SSE-NEXT: movq %xmm1, %rax
; X64-SSE-NEXT: retq
;
; X86-SSE4-LABEL: test_v16i64:
@@ -322,17 +322,17 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE4-NEXT: paddq 56(%ebp), %xmm2
-; X86-SSE4-NEXT: paddq 24(%ebp), %xmm0
-; X86-SSE4-NEXT: paddq %xmm2, %xmm0
; X86-SSE4-NEXT: paddq 72(%ebp), %xmm3
; X86-SSE4-NEXT: paddq 40(%ebp), %xmm1
; X86-SSE4-NEXT: paddq %xmm3, %xmm1
-; X86-SSE4-NEXT: paddq %xmm0, %xmm1
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: paddq 56(%ebp), %xmm2
+; X86-SSE4-NEXT: paddq 24(%ebp), %xmm0
+; X86-SSE4-NEXT: paddq %xmm2, %xmm0
; X86-SSE4-NEXT: paddq %xmm1, %xmm0
-; X86-SSE4-NEXT: movd %xmm0, %eax
-; X86-SSE4-NEXT: pextrd $1, %xmm0, %edx
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm0, %xmm1
+; X86-SSE4-NEXT: movd %xmm1, %eax
+; X86-SSE4-NEXT: pextrd $1, %xmm1, %edx
; X86-SSE4-NEXT: movl %ebp, %esp
; X86-SSE4-NEXT: popl %ebp
; X86-SSE4-NEXT: retl
@@ -343,16 +343,16 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-AVX1-NEXT: movl %esp, %ebp
; X86-AVX1-NEXT: andl $-32, %esp
; X86-AVX1-NEXT: subl $32, %esp
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm3
-; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-NEXT: vpaddq %xmm3, %xmm4, %xmm3
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-NEXT: vpaddq 24(%ebp), %xmm4, %xmm4
+; X86-AVX1-NEXT: vpaddq %xmm4, %xmm3, %xmm3
; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-NEXT: vpaddq 24(%ebp), %xmm1, %xmm1
+; X86-AVX1-NEXT: vpaddq 8(%ebp), %xmm1, %xmm1
; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
+; X86-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
@@ -364,17 +364,17 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
;
; X64-AVX1-LABEL: test_v16i64:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm4
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-NEXT: vpaddq %xmm5, %xmm6, %xmm5
; X64-AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm0, %xmm4, %xmm0
+; X64-AVX1-NEXT: vpaddq %xmm4, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
@@ -418,7 +418,7 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -545,7 +545,7 @@ define i32 @test_v8i32(<8 x i32> %a0) nounwind {
; AVX1-FAST-LABEL: test_v8i32:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
@@ -632,8 +632,7 @@ define i32 @test_v16i32(<16 x i32> %a0) nounwind {
; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: vzeroupper
@@ -655,7 +654,7 @@ define i32 @test_v16i32(<16 x i32> %a0) nounwind {
; AVX512-LABEL: test_v16i32:
; AVX512: # %bb.0:
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -677,36 +676,36 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X86-SSE-NEXT: andl $-16, %esp
; X86-SSE-NEXT: subl $16, %esp
; X86-SSE-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE-NEXT: paddd 56(%ebp), %xmm2
-; X86-SSE-NEXT: paddd 24(%ebp), %xmm0
-; X86-SSE-NEXT: paddd %xmm2, %xmm0
; X86-SSE-NEXT: paddd 72(%ebp), %xmm3
; X86-SSE-NEXT: paddd 40(%ebp), %xmm1
; X86-SSE-NEXT: paddd %xmm3, %xmm1
-; X86-SSE-NEXT: paddd %xmm0, %xmm1
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE-NEXT: paddd 56(%ebp), %xmm2
+; X86-SSE-NEXT: paddd 24(%ebp), %xmm0
+; X86-SSE-NEXT: paddd %xmm2, %xmm0
; X86-SSE-NEXT: paddd %xmm1, %xmm0
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-SSE-NEXT: paddd %xmm0, %xmm1
-; X86-SSE-NEXT: movd %xmm1, %eax
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X86-SSE-NEXT: paddd %xmm1, %xmm0
+; X86-SSE-NEXT: movd %xmm0, %eax
; X86-SSE-NEXT: movl %ebp, %esp
; X86-SSE-NEXT: popl %ebp
; X86-SSE-NEXT: retl
;
; X64-SSE-LABEL: test_v32i32:
; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: paddd %xmm6, %xmm2
-; X64-SSE-NEXT: paddd %xmm4, %xmm0
-; X64-SSE-NEXT: paddd %xmm2, %xmm0
; X64-SSE-NEXT: paddd %xmm7, %xmm3
; X64-SSE-NEXT: paddd %xmm5, %xmm1
; X64-SSE-NEXT: paddd %xmm3, %xmm1
-; X64-SSE-NEXT: paddd %xmm0, %xmm1
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddd %xmm6, %xmm2
+; X64-SSE-NEXT: paddd %xmm4, %xmm0
+; X64-SSE-NEXT: paddd %xmm2, %xmm0
; X64-SSE-NEXT: paddd %xmm1, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-SSE-NEXT: paddd %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X64-SSE-NEXT: paddd %xmm1, %xmm0
+; X64-SSE-NEXT: movd %xmm0, %eax
; X64-SSE-NEXT: retq
;
; X86-AVX1-SLOW-LABEL: test_v32i32:
@@ -715,16 +714,16 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X86-AVX1-SLOW-NEXT: movl %esp, %ebp
; X86-AVX1-SLOW-NEXT: andl $-32, %esp
; X86-AVX1-SLOW-NEXT: subl $32, %esp
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm3
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddd 24(%ebp), %xmm4, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm3, %xmm3
; X86-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpaddd 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-SLOW-NEXT: vpaddd 24(%ebp), %xmm1, %xmm1
+; X86-AVX1-SLOW-NEXT: vpaddd 8(%ebp), %xmm1, %xmm1
; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm2, %xmm0
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm0, %xmm0
; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -737,17 +736,17 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
;
; X64-AVX1-SLOW-LABEL: test_v32i32:
; X64-AVX1-SLOW: # %bb.0:
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm1, %xmm4
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm6, %xmm5
; X64-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm5, %xmm4
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm1, %xmm1
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm4, %xmm0
+; X64-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -762,18 +761,17 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X86-AVX1-FAST-NEXT: movl %esp, %ebp
; X86-AVX1-FAST-NEXT: andl $-32, %esp
; X86-AVX1-FAST-NEXT: subl $32, %esp
-; X86-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm3
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddd 24(%ebp), %xmm4, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm3, %xmm3
; X86-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpaddd 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-FAST-NEXT: vpaddd 24(%ebp), %xmm1, %xmm1
-; X86-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpaddd %xmm0, %xmm2, %xmm0
-; X86-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-FAST-NEXT: vpaddd 8(%ebp), %xmm1, %xmm1
; X86-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X86-AVX1-FAST-NEXT: movl %ebp, %esp
@@ -783,19 +781,18 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
;
; X64-AVX1-FAST-LABEL: test_v32i32:
; X64-AVX1-FAST: # %bb.0:
-; X64-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm4
-; X64-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-FAST-NEXT: vpaddd %xmm5, %xmm6, %xmm5
; X64-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm5, %xmm4
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm1
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vpaddd %xmm0, %xmm4, %xmm0
-; X64-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X64-AVX1-FAST-NEXT: vzeroupper
@@ -841,7 +838,7 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -927,8 +924,7 @@ define i16 @test_v4i16(<4 x i16> %a0) nounwind {
;
; AVX1-FAST-LABEL: test_v4i16:
; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1052,7 +1048,7 @@ define i16 @test_v16i16(<16 x i16> %a0) nounwind {
; AVX1-FAST-LABEL: test_v16i16:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
@@ -1158,10 +1154,8 @@ define i16 @test_v32i16(<32 x i16> %a0) nounwind {
; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm3, %xmm2
; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1187,7 +1181,7 @@ define i16 @test_v32i16(<32 x i16> %a0) nounwind {
; AVX512-LABEL: test_v32i16:
; AVX512: # %bb.0:
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1212,21 +1206,21 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
; X86-SSE-NEXT: andl $-16, %esp
; X86-SSE-NEXT: subl $16, %esp
; X86-SSE-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE-NEXT: paddw 56(%ebp), %xmm2
-; X86-SSE-NEXT: paddw 24(%ebp), %xmm0
-; X86-SSE-NEXT: paddw %xmm2, %xmm0
; X86-SSE-NEXT: paddw 72(%ebp), %xmm3
; X86-SSE-NEXT: paddw 40(%ebp), %xmm1
; X86-SSE-NEXT: paddw %xmm3, %xmm1
-; X86-SSE-NEXT: paddw %xmm0, %xmm1
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE-NEXT: paddw 56(%ebp), %xmm2
+; X86-SSE-NEXT: paddw 24(%ebp), %xmm0
+; X86-SSE-NEXT: paddw %xmm2, %xmm0
; X86-SSE-NEXT: paddw %xmm1, %xmm0
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-SSE-NEXT: paddw %xmm0, %xmm1
-; X86-SSE-NEXT: movdqa %xmm1, %xmm0
-; X86-SSE-NEXT: psrld $16, %xmm0
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; X86-SSE-NEXT: paddw %xmm1, %xmm0
-; X86-SSE-NEXT: movd %xmm0, %eax
+; X86-SSE-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE-NEXT: psrld $16, %xmm1
+; X86-SSE-NEXT: paddw %xmm0, %xmm1
+; X86-SSE-NEXT: movd %xmm1, %eax
; X86-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE-NEXT: movl %ebp, %esp
; X86-SSE-NEXT: popl %ebp
@@ -1234,21 +1228,21 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
;
; X64-SSE-LABEL: test_v64i16:
; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: paddw %xmm6, %xmm2
-; X64-SSE-NEXT: paddw %xmm4, %xmm0
-; X64-SSE-NEXT: paddw %xmm2, %xmm0
; X64-SSE-NEXT: paddw %xmm7, %xmm3
; X64-SSE-NEXT: paddw %xmm5, %xmm1
; X64-SSE-NEXT: paddw %xmm3, %xmm1
-; X64-SSE-NEXT: paddw %xmm0, %xmm1
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddw %xmm6, %xmm2
+; X64-SSE-NEXT: paddw %xmm4, %xmm0
+; X64-SSE-NEXT: paddw %xmm2, %xmm0
; X64-SSE-NEXT: paddw %xmm1, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-SSE-NEXT: paddw %xmm0, %xmm1
-; X64-SSE-NEXT: movdqa %xmm1, %xmm0
-; X64-SSE-NEXT: psrld $16, %xmm0
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; X64-SSE-NEXT: paddw %xmm1, %xmm0
-; X64-SSE-NEXT: movd %xmm0, %eax
+; X64-SSE-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE-NEXT: psrld $16, %xmm1
+; X64-SSE-NEXT: paddw %xmm0, %xmm1
+; X64-SSE-NEXT: movd %xmm1, %eax
; X64-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SSE-NEXT: retq
;
@@ -1258,16 +1252,16 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
; X86-AVX1-SLOW-NEXT: movl %esp, %ebp
; X86-AVX1-SLOW-NEXT: andl $-32, %esp
; X86-AVX1-SLOW-NEXT: subl $32, %esp
-; X86-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm3
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm4, %xmm3
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddw 24(%ebp), %xmm4, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm3, %xmm3
; X86-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpaddw 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm3, %xmm2
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-SLOW-NEXT: vpaddw 24(%ebp), %xmm1, %xmm1
+; X86-AVX1-SLOW-NEXT: vpaddw 8(%ebp), %xmm1, %xmm1
; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm0, %xmm0
; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1283,17 +1277,17 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
;
; X64-AVX1-SLOW-LABEL: test_v64i16:
; X64-AVX1-SLOW: # %bb.0:
-; X64-AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm1, %xmm4
-; X64-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm5, %xmm6, %xmm5
; X64-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm5, %xmm4
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm1, %xmm1
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vpaddw %xmm0, %xmm4, %xmm0
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1311,20 +1305,18 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
; X86-AVX1-FAST-NEXT: movl %esp, %ebp
; X86-AVX1-FAST-NEXT: andl $-32, %esp
; X86-AVX1-FAST-NEXT: subl $32, %esp
-; X86-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm3
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddw %xmm3, %xmm4, %xmm3
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddw 24(%ebp), %xmm4, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm3, %xmm3
; X86-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpaddw 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm3, %xmm2
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-FAST-NEXT: vpaddw 24(%ebp), %xmm1, %xmm1
-; X86-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpaddw %xmm0, %xmm2, %xmm0
-; X86-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-AVX1-FAST-NEXT: vpaddw 8(%ebp), %xmm1, %xmm1
; X86-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vpaddw %xmm3, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X86-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1335,21 +1327,19 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
;
; X64-AVX1-FAST-LABEL: test_v64i16:
; X64-AVX1-FAST: # %bb.0:
-; X64-AVX1-FAST-NEXT: vpaddw %xmm3, %xmm1, %xmm4
-; X64-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-FAST-NEXT: vpaddw %xmm5, %xmm6, %xmm5
; X64-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm5, %xmm4
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-FAST-NEXT: vpaddw %xmm3, %xmm1, %xmm1
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vpaddw %xmm0, %xmm4, %xmm0
-; X64-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X64-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1402,7 +1392,7 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1424,27 +1414,47 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
;
define i8 @test_v2i8(<2 x i8> %a0) nounwind {
-; SSE-LABEL: test_v2i8:
-; SSE: # %bb.0:
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrlw $8, %xmm1
-; SSE-NEXT: paddb %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: # kill: def $al killed $al killed $eax
-; SSE-NEXT: ret{{[l|q]}}
+; X86-SSE2-LABEL: test_v2i8:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: # kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT: retl
+;
+; X64-SSE2-LABEL: test_v2i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE2-NEXT: movd %xmm1, %eax
+; X64-SSE2-NEXT: # kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT: retq
+;
+; SSE4-LABEL: test_v2i8:
+; SSE4: # %bb.0:
+; SSE4-NEXT: pxor %xmm1, %xmm1
+; SSE4-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; SSE4-NEXT: psadbw %xmm1, %xmm0
+; SSE4-NEXT: movd %xmm0, %eax
+; SSE4-NEXT: # kill: def $al killed $al killed $eax
+; SSE4-NEXT: ret{{[l|q]}}
;
; AVX-LABEL: test_v2i8:
; AVX: # %bb.0:
-; AVX-NEXT: vpsrlw $8, %xmm0, %xmm1
-; AVX-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX-NEXT: vmovd %xmm0, %eax
; AVX-NEXT: # kill: def $al killed $al killed $eax
; AVX-NEXT: ret{{[l|q]}}
;
; AVX512-LABEL: test_v2i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: # kill: def $al killed $al killed $eax
; AVX512-NEXT: retq
@@ -1453,36 +1463,60 @@ define i8 @test_v2i8(<2 x i8> %a0) nounwind {
}
define i8 @test_v2i8_load(ptr %p) nounwind {
-; X86-SSE-LABEL: test_v2i8_load:
-; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT: movzwl (%eax), %eax
-; X86-SSE-NEXT: movd %eax, %xmm0
-; X86-SSE-NEXT: movdqa %xmm0, %xmm1
-; X86-SSE-NEXT: psrlw $8, %xmm1
-; X86-SSE-NEXT: paddb %xmm0, %xmm1
-; X86-SSE-NEXT: movd %xmm1, %eax
-; X86-SSE-NEXT: # kill: def $al killed $al killed $eax
-; X86-SSE-NEXT: retl
+; X86-SSE2-LABEL: test_v2i8_load:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: movzwl (%eax), %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: # kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v2i8_load:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: movzwl (%rdi), %eax
-; X64-SSE-NEXT: movd %eax, %xmm0
-; X64-SSE-NEXT: movdqa %xmm0, %xmm1
-; X64-SSE-NEXT: psrlw $8, %xmm1
-; X64-SSE-NEXT: paddb %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
-; X64-SSE-NEXT: # kill: def $al killed $al killed $eax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v2i8_load:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: movzwl (%rdi), %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE2-NEXT: movd %xmm1, %eax
+; X64-SSE2-NEXT: # kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: test_v2i8_load:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE4-NEXT: movzwl (%eax), %eax
+; X86-SSE4-NEXT: movd %eax, %xmm0
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: # kill: def $al killed $al killed $eax
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: test_v2i8_load:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: movzwl (%rdi), %eax
+; X64-SSE4-NEXT: movd %eax, %xmm0
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE4-NEXT: movd %xmm0, %eax
+; X64-SSE4-NEXT: # kill: def $al killed $al killed $eax
+; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: test_v2i8_load:
; X86-AVX1: # %bb.0:
; X86-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-AVX1-NEXT: movzwl (%eax), %eax
; X86-AVX1-NEXT: vmovd %eax, %xmm0
-; X86-AVX1-NEXT: vpsrlw $8, %xmm0, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: # kill: def $al killed $al killed $eax
; X86-AVX1-NEXT: retl
@@ -1491,8 +1525,9 @@ define i8 @test_v2i8_load(ptr %p) nounwind {
; X64-AVX1: # %bb.0:
; X64-AVX1-NEXT: movzwl (%rdi), %eax
; X64-AVX1-NEXT: vmovd %eax, %xmm0
-; X64-AVX1-NEXT: vpsrlw $8, %xmm0, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovd %xmm0, %eax
; X64-AVX1-NEXT: # kill: def $al killed $al killed $eax
; X64-AVX1-NEXT: retq
@@ -1502,8 +1537,9 @@ define i8 @test_v2i8_load(ptr %p) nounwind {
; X86-AVX2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-AVX2-NEXT: movzwl (%eax), %eax
; X86-AVX2-NEXT: vmovd %eax, %xmm0
-; X86-AVX2-NEXT: vpsrlw $8, %xmm0, %xmm1
-; X86-AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: # kill: def $al killed $al killed $eax
; X86-AVX2-NEXT: retl
@@ -1512,8 +1548,9 @@ define i8 @test_v2i8_load(ptr %p) nounwind {
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: movzwl (%rdi), %eax
; X64-AVX2-NEXT: vmovd %eax, %xmm0
-; X64-AVX2-NEXT: vpsrlw $8, %xmm0, %xmm1
-; X64-AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovd %xmm0, %eax
; X64-AVX2-NEXT: # kill: def $al killed $al killed $eax
; X64-AVX2-NEXT: retq
@@ -1522,8 +1559,9 @@ define i8 @test_v2i8_load(ptr %p) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: movzwl (%rdi), %eax
; AVX512-NEXT: vmovd %eax, %xmm0
-; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: # kill: def $al killed $al killed $eax
; AVX512-NEXT: retq
@@ -1536,9 +1574,10 @@ define i8 @test_v4i8(<4 x i8> %a0) nounwind {
; SSE2-LABEL: test_v4i8:
; SSE2: # %bb.0:
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: psadbw %xmm1, %xmm0
-; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: xorps %xmm2, %xmm2
+; SSE2-NEXT: movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
+; SSE2-NEXT: psadbw %xmm1, %xmm2
+; SSE2-NEXT: movd %xmm2, %eax
; SSE2-NEXT: # kill: def $al killed $al killed $eax
; SSE2-NEXT: ret{{[l|q]}}
;
@@ -2094,6 +2133,3 @@ declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)
declare i8 @llvm.vector.reduce.add.v32i8(<32 x i8>)
declare i8 @llvm.vector.reduce.add.v64i8(<64 x i8>)
declare i8 @llvm.vector.reduce.add.v128i8(<128 x i8>)
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; X64-SSE2: {{.*}}
-; X64-SSE4: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll b/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll
index e936f1e4faf21..54c58fa928acd 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll
@@ -228,11 +228,10 @@ define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) nounwind {
; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
; X86-SSE2-NEXT: packuswb %xmm2, %xmm1
-; X86-SSE2-NEXT: packuswb %xmm2, %xmm2
-; X86-SSE2-NEXT: paddd %xmm1, %xmm2
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
-; X86-SSE2-NEXT: paddd %xmm2, %xmm0
-; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: packuswb %xmm0, %xmm1
+; X86-SSE2-NEXT: packuswb %xmm0, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE2-NEXT: movd %xmm1, %eax
; X86-SSE2-NEXT: retl
;
; X64-SSE2-LABEL: reduce_ctpop_v4i32:
@@ -258,11 +257,10 @@ define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) nounwind {
; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
; X64-SSE2-NEXT: packuswb %xmm2, %xmm1
-; X64-SSE2-NEXT: packuswb %xmm2, %xmm2
-; X64-SSE2-NEXT: paddd %xmm1, %xmm2
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
-; X64-SSE2-NEXT: paddd %xmm2, %xmm0
-; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: packuswb %xmm0, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm0, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE2-NEXT: movd %xmm1, %eax
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: reduce_ctpop_v4i32:
@@ -283,11 +281,10 @@ define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) nounwind {
; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
; X86-SSE4-NEXT: psadbw %xmm0, %xmm2
; X86-SSE4-NEXT: packuswb %xmm1, %xmm2
-; X86-SSE4-NEXT: packuswb %xmm1, %xmm1
-; X86-SSE4-NEXT: paddd %xmm2, %xmm1
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X86-SSE4-NEXT: paddd %xmm1, %xmm0
-; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: packusdw %xmm0, %xmm2
+; X86-SSE4-NEXT: packuswb %xmm0, %xmm2
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm2
+; X86-SSE4-NEXT: movd %xmm2, %eax
; X86-SSE4-NEXT: retl
;
; X64-SSE4-LABEL: reduce_ctpop_v4i32:
@@ -308,11 +305,10 @@ define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) nounwind {
; X64-SSE4-NEXT: psadbw %xmm0, %xmm3
; X64-SSE4-NEXT: psadbw %xmm0, %xmm1
; X64-SSE4-NEXT: packuswb %xmm3, %xmm1
-; X64-SSE4-NEXT: packuswb %xmm3, %xmm3
-; X64-SSE4-NEXT: paddd %xmm1, %xmm3
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
-; X64-SSE4-NEXT: paddd %xmm3, %xmm0
-; X64-SSE4-NEXT: movd %xmm0, %eax
+; X64-SSE4-NEXT: packusdw %xmm0, %xmm1
+; X64-SSE4-NEXT: packuswb %xmm0, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE4-NEXT: movd %xmm1, %eax
; X64-SSE4-NEXT: retq
;
; AVX1-LABEL: reduce_ctpop_v4i32:
@@ -331,10 +327,9 @@ define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) nounwind {
; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpackuswb %xmm2, %xmm2, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vmovd %xmm0, %eax
; AVX1-NEXT: ret{{[l|q]}}
;
@@ -354,11 +349,13 @@ define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) nounwind {
; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: vpackuswb %xmm2, %xmm2, %xmm1
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpackusdw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm2
+; AVX2-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: ret{{[l|q]}}
;
; AVX512VL-LABEL: reduce_ctpop_v4i32:
@@ -377,18 +374,22 @@ define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) nounwind {
; AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT: vpmovdb %xmm0, %xmm0
+; AVX512VL-NEXT: vpmovdb %zmm0, %xmm0
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovd %xmm0, %eax
+; AVX512VL-NEXT: vmovq %xmm0, %rax
+; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VPOPCNT-LABEL: reduce_ctpop_v4i32:
; AVX512VPOPCNT: # %bb.0:
; AVX512VPOPCNT-NEXT: vpopcntd %xmm0, %xmm0
-; AVX512VPOPCNT-NEXT: vpmovdb %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpmovdb %zmm0, %xmm0
; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax
+; AVX512VPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX512VPOPCNT-NEXT: vzeroupper
; AVX512VPOPCNT-NEXT: retq
%p0 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %a0)
%r0 = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %p0)
@@ -416,10 +417,10 @@ define i16 @reduce_ctpop_v8i16(<8 x i16> %a0) nounwind {
; X86-SSE2-NEXT: psllw $8, %xmm0
; X86-SSE2-NEXT: paddb %xmm1, %xmm0
; X86-SSE2-NEXT: psrlw $8, %xmm0
-; X86-SSE2-NEXT: packuswb %xmm0, %xmm0
; X86-SSE2-NEXT: pxor %xmm1, %xmm1
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
; X86-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE2-NEXT: retl
;
@@ -443,10 +444,10 @@ define i16 @reduce_ctpop_v8i16(<8 x i16> %a0) nounwind {
; X64-SSE2-NEXT: psllw $8, %xmm0
; X64-SSE2-NEXT: paddb %xmm1, %xmm0
; X64-SSE2-NEXT: psrlw $8, %xmm0
-; X64-SSE2-NEXT: packuswb %xmm0, %xmm0
; X64-SSE2-NEXT: pxor %xmm1, %xmm1
-; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
-; X64-SSE2-NEXT: movd %xmm1, %eax
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: movd %xmm0, %eax
; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SSE2-NEXT: retq
;
@@ -466,52 +467,92 @@ define i16 @reduce_ctpop_v8i16(<8 x i16> %a0) nounwind {
; SSE4-NEXT: psllw $8, %xmm0
; SSE4-NEXT: paddb %xmm3, %xmm0
; SSE4-NEXT: psrlw $8, %xmm0
-; SSE4-NEXT: packuswb %xmm0, %xmm0
; SSE4-NEXT: pxor %xmm1, %xmm1
-; SSE4-NEXT: psadbw %xmm0, %xmm1
-; SSE4-NEXT: movd %xmm1, %eax
+; SSE4-NEXT: packuswb %xmm1, %xmm0
+; SSE4-NEXT: psadbw %xmm1, %xmm0
+; SSE4-NEXT: movd %xmm0, %eax
; SSE4-NEXT: # kill: def $ax killed $ax killed $eax
; SSE4-NEXT: ret{{[l|q]}}
;
-; AVX1-LABEL: reduce_ctpop_v8i16:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpsllw $8, %xmm0, %xmm1
-; AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX1-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX1-NEXT: ret{{[l|q]}}
+; X86-AVX1-LABEL: reduce_ctpop_v8i16:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsllw $8, %xmm0, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
+; X86-AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-NEXT: retl
;
-; AVX2-LABEL: reduce_ctpop_v8i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: vpsllw $8, %xmm0, %xmm1
-; AVX2-NEXT: vpaddb %xmm0, %xmm1, %xmm0
-; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX2-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: ret{{[l|q]}}
+; X64-AVX1-LABEL: reduce_ctpop_v8i16:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; X64-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsllw $8, %xmm0, %xmm1
+; X64-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
+; X64-AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX1-NEXT: retq
+;
+; X86-AVX2-LABEL: reduce_ctpop_v8i16:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X86-AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X86-AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; X86-AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpsllw $8, %xmm0, %xmm1
+; X86-AVX2-NEXT: vpaddb %xmm0, %xmm1, %xmm0
+; X86-AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: reduce_ctpop_v8i16:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X64-AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; X64-AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpsllw $8, %xmm0, %xmm1
+; X64-AVX2-NEXT: vpaddb %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX2-NEXT: retq
;
; AVX512VL-LABEL: reduce_ctpop_v8i16:
; AVX512VL: # %bb.0:
@@ -526,22 +567,24 @@ define i16 @reduce_ctpop_v8i16(<8 x i16> %a0) nounwind {
; AVX512VL-NEXT: vpsllw $8, %xmm0, %xmm1
; AVX512VL-NEXT: vpaddb %xmm0, %xmm1, %xmm0
; AVX512VL-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX512VL-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
+; AVX512VL-NEXT: vpmovwb %ymm0, %xmm0
; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovd %xmm0, %eax
-; AVX512VL-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512VL-NEXT: vmovq %xmm0, %rax
+; AVX512VL-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VPOPCNT-LABEL: reduce_ctpop_v8i16:
; AVX512VPOPCNT: # %bb.0:
; AVX512VPOPCNT-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX512VPOPCNT-NEXT: vpopcntd %ymm0, %ymm0
-; AVX512VPOPCNT-NEXT: vpmovdb %ymm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpmovdw %ymm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpmovwb %ymm0, %xmm0
; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax
-; AVX512VPOPCNT-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax
+; AVX512VPOPCNT-NEXT: # kill: def $ax killed $ax killed $rax
; AVX512VPOPCNT-NEXT: vzeroupper
; AVX512VPOPCNT-NEXT: retq
%p0 = tail call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %a0)
@@ -707,12 +750,14 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; X86-SSE2-NEXT: movdqa %xmm1, %xmm2
; X86-SSE2-NEXT: psrlw $4, %xmm2
; X86-SSE2-NEXT: paddb %xmm1, %xmm2
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE2-NEXT: pand %xmm1, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
-; X86-SSE2-NEXT: psrlw $1, %xmm5
-; X86-SSE2-NEXT: pand %xmm4, %xmm5
-; X86-SSE2-NEXT: psubb %xmm5, %xmm0
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm5, %xmm2
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm6
+; X86-SSE2-NEXT: psrlw $1, %xmm6
+; X86-SSE2-NEXT: pand %xmm4, %xmm6
+; X86-SSE2-NEXT: psubb %xmm6, %xmm0
; X86-SSE2-NEXT: movdqa %xmm0, %xmm4
; X86-SSE2-NEXT: pand %xmm3, %xmm4
; X86-SSE2-NEXT: psrlw $2, %xmm0
@@ -721,15 +766,14 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
; X86-SSE2-NEXT: psrlw $4, %xmm3
; X86-SSE2-NEXT: paddb %xmm0, %xmm3
-; X86-SSE2-NEXT: pand %xmm1, %xmm3
-; X86-SSE2-NEXT: paddb %xmm2, %xmm3
-; X86-SSE2-NEXT: pxor %xmm0, %xmm0
-; X86-SSE2-NEXT: psadbw %xmm3, %xmm0
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm0, %xmm1
-; X86-SSE2-NEXT: movd %xmm1, %eax
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X86-SSE2-NEXT: movd %xmm0, %edx
+; X86-SSE2-NEXT: pand %xmm5, %xmm3
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm3
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm3
+; X86-SSE2-NEXT: movd %xmm3, %eax
+; X86-SSE2-NEXT: xorl %edx, %edx
; X86-SSE2-NEXT: retl
;
; X64-SSE2-LABEL: reduce_ctpop_v4i64:
@@ -739,36 +783,38 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; X64-SSE2-NEXT: pand %xmm3, %xmm2
; X64-SSE2-NEXT: psubb %xmm2, %xmm1
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm4
-; X64-SSE2-NEXT: pand %xmm2, %xmm4
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X64-SSE2-NEXT: pand %xmm4, %xmm2
; X64-SSE2-NEXT: psrlw $2, %xmm1
-; X64-SSE2-NEXT: pand %xmm2, %xmm1
-; X64-SSE2-NEXT: paddb %xmm4, %xmm1
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm4
-; X64-SSE2-NEXT: psrlw $4, %xmm4
-; X64-SSE2-NEXT: paddb %xmm1, %xmm4
+; X64-SSE2-NEXT: pand %xmm4, %xmm1
+; X64-SSE2-NEXT: paddb %xmm2, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X64-SSE2-NEXT: psrlw $4, %xmm2
+; X64-SSE2-NEXT: paddb %xmm1, %xmm2
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-SSE2-NEXT: pand %xmm1, %xmm4
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm5
-; X64-SSE2-NEXT: psrlw $1, %xmm5
-; X64-SSE2-NEXT: pand %xmm3, %xmm5
-; X64-SSE2-NEXT: psubb %xmm5, %xmm0
+; X64-SSE2-NEXT: pand %xmm1, %xmm2
+; X64-SSE2-NEXT: pxor %xmm5, %xmm5
+; X64-SSE2-NEXT: psadbw %xmm5, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm6
+; X64-SSE2-NEXT: psrlw $1, %xmm6
+; X64-SSE2-NEXT: pand %xmm3, %xmm6
+; X64-SSE2-NEXT: psubb %xmm6, %xmm0
; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE2-NEXT: pand %xmm2, %xmm3
+; X64-SSE2-NEXT: pand %xmm4, %xmm3
; X64-SSE2-NEXT: psrlw $2, %xmm0
-; X64-SSE2-NEXT: pand %xmm2, %xmm0
+; X64-SSE2-NEXT: pand %xmm4, %xmm0
; X64-SSE2-NEXT: paddb %xmm3, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE2-NEXT: psrlw $4, %xmm2
-; X64-SSE2-NEXT: paddb %xmm0, %xmm2
-; X64-SSE2-NEXT: pand %xmm1, %xmm2
-; X64-SSE2-NEXT: paddb %xmm4, %xmm2
-; X64-SSE2-NEXT: pxor %xmm0, %xmm0
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm0
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm0, %xmm1
-; X64-SSE2-NEXT: movq %xmm1, %rax
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE2-NEXT: psrlw $4, %xmm3
+; X64-SSE2-NEXT: paddb %xmm0, %xmm3
+; X64-SSE2-NEXT: pand %xmm1, %xmm3
+; X64-SSE2-NEXT: psadbw %xmm5, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm2, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm5, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm5, %xmm3
+; X64-SSE2-NEXT: psadbw %xmm5, %xmm3
+; X64-SSE2-NEXT: movq %xmm3, %rax
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: reduce_ctpop_v4i64:
@@ -784,50 +830,54 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; X86-SSE4-NEXT: movdqa %xmm2, %xmm4
; X86-SSE4-NEXT: pshufb %xmm1, %xmm4
; X86-SSE4-NEXT: paddb %xmm5, %xmm4
-; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
-; X86-SSE4-NEXT: pand %xmm3, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm5
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm4
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm5
+; X86-SSE4-NEXT: pand %xmm3, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm2, %xmm6
+; X86-SSE4-NEXT: pshufb %xmm5, %xmm6
; X86-SSE4-NEXT: psrlw $4, %xmm0
; X86-SSE4-NEXT: pand %xmm3, %xmm0
; X86-SSE4-NEXT: pshufb %xmm0, %xmm2
-; X86-SSE4-NEXT: paddb %xmm5, %xmm2
-; X86-SSE4-NEXT: paddb %xmm4, %xmm2
-; X86-SSE4-NEXT: pxor %xmm0, %xmm0
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm0
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm0, %xmm1
-; X86-SSE4-NEXT: movd %xmm1, %eax
-; X86-SSE4-NEXT: pextrd $1, %xmm1, %edx
+; X86-SSE4-NEXT: paddb %xmm6, %xmm2
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm2
+; X86-SSE4-NEXT: packusdw %xmm4, %xmm2
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm2
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm2
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm2
+; X86-SSE4-NEXT: movd %xmm2, %eax
+; X86-SSE4-NEXT: pextrd $1, %xmm2, %edx
; X86-SSE4-NEXT: retl
;
; X64-SSE4-LABEL: reduce_ctpop_v4i64:
; X64-SSE4: # %bb.0:
-; X64-SSE4-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-SSE4-NEXT: movdqa %xmm1, %xmm3
-; X64-SSE4-NEXT: pand %xmm2, %xmm3
-; X64-SSE4-NEXT: movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X64-SSE4-NEXT: movdqa %xmm4, %xmm5
-; X64-SSE4-NEXT: pshufb %xmm3, %xmm5
+; X64-SSE4-NEXT: movdqa {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm4
+; X64-SSE4-NEXT: pand %xmm3, %xmm4
+; X64-SSE4-NEXT: movdqa {{.*#+}} xmm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-SSE4-NEXT: movdqa %xmm2, %xmm5
+; X64-SSE4-NEXT: pshufb %xmm4, %xmm5
; X64-SSE4-NEXT: psrlw $4, %xmm1
-; X64-SSE4-NEXT: pand %xmm2, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm4, %xmm3
-; X64-SSE4-NEXT: pshufb %xmm1, %xmm3
-; X64-SSE4-NEXT: paddb %xmm5, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm1
-; X64-SSE4-NEXT: pand %xmm2, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm4, %xmm5
-; X64-SSE4-NEXT: pshufb %xmm1, %xmm5
-; X64-SSE4-NEXT: psrlw $4, %xmm0
-; X64-SSE4-NEXT: pand %xmm2, %xmm0
-; X64-SSE4-NEXT: pshufb %xmm0, %xmm4
+; X64-SSE4-NEXT: pand %xmm3, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm2, %xmm4
+; X64-SSE4-NEXT: pshufb %xmm1, %xmm4
; X64-SSE4-NEXT: paddb %xmm5, %xmm4
-; X64-SSE4-NEXT: paddb %xmm3, %xmm4
-; X64-SSE4-NEXT: pxor %xmm0, %xmm0
-; X64-SSE4-NEXT: psadbw %xmm4, %xmm0
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm0, %xmm1
-; X64-SSE4-NEXT: movq %xmm1, %rax
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm4
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm5
+; X64-SSE4-NEXT: pand %xmm3, %xmm5
+; X64-SSE4-NEXT: movdqa %xmm2, %xmm6
+; X64-SSE4-NEXT: pshufb %xmm5, %xmm6
+; X64-SSE4-NEXT: psrlw $4, %xmm0
+; X64-SSE4-NEXT: pand %xmm3, %xmm0
+; X64-SSE4-NEXT: pshufb %xmm0, %xmm2
+; X64-SSE4-NEXT: paddb %xmm6, %xmm2
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm2
+; X64-SSE4-NEXT: packusdw %xmm4, %xmm2
+; X64-SSE4-NEXT: packusdw %xmm1, %xmm2
+; X64-SSE4-NEXT: packuswb %xmm1, %xmm2
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm2
+; X64-SSE4-NEXT: movq %xmm2, %rax
; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: reduce_ctpop_v4i64:
@@ -841,17 +891,19 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; X86-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
; X86-AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
-; X86-AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; X86-AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm5
+; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: vpextrd $1, %xmm0, %edx
; X86-AVX1-NEXT: vzeroupper
@@ -868,17 +920,19 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; X64-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
; X64-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
; X64-AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
-; X64-AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
+; X64-AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm5
+; X64-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
; X64-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm3, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
; X64-AVX1-NEXT: vzeroupper
; X64-AVX1-NEXT: retq
@@ -896,10 +950,14 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; X86-AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0
; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: vpextrd $1, %xmm0, %edx
; X86-AVX2-NEXT: vzeroupper
@@ -918,10 +976,14 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; X64-AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0
; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; X64-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovq %xmm0, %rax
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
@@ -939,7 +1001,7 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; AVX512VL-NEXT: vpaddb %ymm2, %ymm0, %ymm0
; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT: vpmovqb %ymm0, %xmm0
+; AVX512VL-NEXT: vpmovqb %zmm0, %xmm0
; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vmovq %xmm0, %rax
@@ -949,7 +1011,7 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; AVX512VPOPCNT-LABEL: reduce_ctpop_v4i64:
; AVX512VPOPCNT: # %bb.0:
; AVX512VPOPCNT-NEXT: vpopcntq %ymm0, %ymm0
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax
@@ -1005,12 +1067,10 @@ define i32 @reduce_ctpop_v8i32(<8 x i32> %a0) nounwind {
; SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
; SSE2-NEXT: psadbw %xmm1, %xmm3
; SSE2-NEXT: packuswb %xmm0, %xmm3
-; SSE2-NEXT: paddd %xmm2, %xmm3
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
-; SSE2-NEXT: paddd %xmm3, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: packuswb %xmm2, %xmm3
+; SSE2-NEXT: packuswb %xmm1, %xmm3
+; SSE2-NEXT: psadbw %xmm1, %xmm3
+; SSE2-NEXT: movd %xmm3, %eax
; SSE2-NEXT: ret{{[l|q]}}
;
; X86-SSE4-LABEL: reduce_ctpop_v8i32:
@@ -1026,12 +1086,12 @@ define i32 @reduce_ctpop_v8i32(<8 x i32> %a0) nounwind {
; X86-SSE4-NEXT: movdqa %xmm2, %xmm6
; X86-SSE4-NEXT: pshufb %xmm1, %xmm6
; X86-SSE4-NEXT: paddb %xmm5, %xmm6
-; X86-SSE4-NEXT: pxor %xmm4, %xmm4
-; X86-SSE4-NEXT: pmovzxdq {{.*#+}} xmm1 = xmm6[0],zero,xmm6[1],zero
-; X86-SSE4-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; X86-SSE4-NEXT: psadbw %xmm4, %xmm6
-; X86-SSE4-NEXT: psadbw %xmm4, %xmm1
-; X86-SSE4-NEXT: packuswb %xmm6, %xmm1
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: pmovzxdq {{.*#+}} xmm4 = xmm6[0],zero,xmm6[1],zero
+; X86-SSE4-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm1[2],xmm6[3],xmm1[3]
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm6
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm4
+; X86-SSE4-NEXT: packuswb %xmm6, %xmm4
; X86-SSE4-NEXT: movdqa %xmm0, %xmm5
; X86-SSE4-NEXT: pand %xmm3, %xmm5
; X86-SSE4-NEXT: movdqa %xmm2, %xmm6
@@ -1041,15 +1101,13 @@ define i32 @reduce_ctpop_v8i32(<8 x i32> %a0) nounwind {
; X86-SSE4-NEXT: pshufb %xmm0, %xmm2
; X86-SSE4-NEXT: paddb %xmm6, %xmm2
; X86-SSE4-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm2[0],zero,xmm2[1],zero
-; X86-SSE4-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
-; X86-SSE4-NEXT: psadbw %xmm4, %xmm2
-; X86-SSE4-NEXT: psadbw %xmm4, %xmm0
+; X86-SSE4-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm2
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm0
; X86-SSE4-NEXT: packuswb %xmm2, %xmm0
-; X86-SSE4-NEXT: paddd %xmm1, %xmm0
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE4-NEXT: paddd %xmm0, %xmm1
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X86-SSE4-NEXT: paddd %xmm1, %xmm0
+; X86-SSE4-NEXT: packusdw %xmm4, %xmm0
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm0
; X86-SSE4-NEXT: movd %xmm0, %eax
; X86-SSE4-NEXT: retl
;
@@ -1085,47 +1143,43 @@ define i32 @reduce_ctpop_v8i32(<8 x i32> %a0) nounwind {
; X64-SSE4-NEXT: psadbw %xmm1, %xmm2
; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
; X64-SSE4-NEXT: packuswb %xmm2, %xmm0
-; X64-SSE4-NEXT: paddd %xmm5, %xmm0
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE4-NEXT: paddd %xmm0, %xmm1
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X64-SSE4-NEXT: paddd %xmm1, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm5, %xmm0
+; X64-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
; X64-SSE4-NEXT: movd %xmm0, %eax
; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: reduce_ctpop_v8i32:
; X86-AVX1: # %bb.0:
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-AVX1-NEXT: vpand %xmm1, %xmm3, %xmm4
-; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X86-AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm4
-; X86-AVX1-NEXT: vpsrlw $4, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpand %xmm1, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm3
-; X86-AVX1-NEXT: vpaddb %xmm4, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpackuswb %xmm5, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm2, %xmm5
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm4
+; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX1-NEXT: vpshufb %xmm4, %xmm3, %xmm4
+; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm3, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm4, %xmm1, %xmm4
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpackuswb %xmm5, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm5
+; X86-AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm5
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm0
+; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
; X86-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm2, %xmm2
; X86-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm4, %xmm0, %xmm0
; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: vzeroupper
; X86-AVX1-NEXT: retl
@@ -1158,11 +1212,9 @@ define i32 @reduce_ctpop_v8i32(<8 x i32> %a0) nounwind {
; X64-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
; X64-AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovd %xmm0, %eax
; X64-AVX1-NEXT: vzeroupper
; X64-AVX1-NEXT: retq
@@ -1184,12 +1236,12 @@ define i32 @reduce_ctpop_v8i32(<8 x i32> %a0) nounwind {
; AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: ret{{[l|q]}}
@@ -1211,20 +1263,22 @@ define i32 @reduce_ctpop_v8i32(<8 x i32> %a0) nounwind {
; AVX512VL-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT: vpmovdb %ymm0, %xmm0
+; AVX512VL-NEXT: vpmovdb %zmm0, %xmm0
; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovd %xmm0, %eax
+; AVX512VL-NEXT: vmovq %xmm0, %rax
+; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VPOPCNT-LABEL: reduce_ctpop_v8i32:
; AVX512VPOPCNT: # %bb.0:
; AVX512VPOPCNT-NEXT: vpopcntd %ymm0, %ymm0
-; AVX512VPOPCNT-NEXT: vpmovdb %ymm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpmovdb %zmm0, %xmm0
; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax
+; AVX512VPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VPOPCNT-NEXT: vzeroupper
; AVX512VPOPCNT-NEXT: retq
%p0 = tail call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %a0)
@@ -1240,22 +1294,22 @@ define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) nounwind {
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm5
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm3
; X86-SSE2-NEXT: psrlw $1, %xmm3
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; X86-SSE2-NEXT: pand %xmm4, %xmm3
-; X86-SSE2-NEXT: psubb %xmm3, %xmm2
+; X86-SSE2-NEXT: psubb %xmm3, %xmm1
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm6
; X86-SSE2-NEXT: pand %xmm3, %xmm6
-; X86-SSE2-NEXT: psrlw $2, %xmm2
-; X86-SSE2-NEXT: pand %xmm3, %xmm2
-; X86-SSE2-NEXT: paddb %xmm6, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm7
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm3, %xmm1
+; X86-SSE2-NEXT: paddb %xmm6, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm7
; X86-SSE2-NEXT: psrlw $4, %xmm7
-; X86-SSE2-NEXT: paddb %xmm2, %xmm7
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE2-NEXT: pand %xmm2, %xmm7
+; X86-SSE2-NEXT: paddb %xmm1, %xmm7
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm1, %xmm7
; X86-SSE2-NEXT: movdqa %xmm0, %xmm6
; X86-SSE2-NEXT: psrlw $1, %xmm6
; X86-SSE2-NEXT: pand %xmm4, %xmm6
@@ -1268,79 +1322,84 @@ define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) nounwind {
; X86-SSE2-NEXT: movdqa %xmm0, %xmm6
; X86-SSE2-NEXT: psrlw $4, %xmm6
; X86-SSE2-NEXT: paddb %xmm0, %xmm6
-; X86-SSE2-NEXT: pand %xmm2, %xmm6
-; X86-SSE2-NEXT: paddb %xmm7, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm0
-; X86-SSE2-NEXT: psrlw $1, %xmm0
-; X86-SSE2-NEXT: pand %xmm4, %xmm0
-; X86-SSE2-NEXT: psubb %xmm0, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: pxor %xmm0, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm7
+; X86-SSE2-NEXT: pand %xmm1, %xmm6
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm6
+; X86-SSE2-NEXT: packuswb %xmm7, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm5, %xmm7
+; X86-SSE2-NEXT: psrlw $1, %xmm7
+; X86-SSE2-NEXT: pand %xmm4, %xmm7
+; X86-SSE2-NEXT: psubb %xmm7, %xmm5
+; X86-SSE2-NEXT: movdqa %xmm5, %xmm7
+; X86-SSE2-NEXT: pand %xmm3, %xmm7
; X86-SSE2-NEXT: psrlw $2, %xmm5
; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: paddb %xmm0, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm0
-; X86-SSE2-NEXT: psrlw $4, %xmm0
-; X86-SSE2-NEXT: paddb %xmm5, %xmm0
-; X86-SSE2-NEXT: pand %xmm2, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm5
+; X86-SSE2-NEXT: paddb %xmm7, %xmm5
+; X86-SSE2-NEXT: movdqa %xmm5, %xmm7
+; X86-SSE2-NEXT: psrlw $4, %xmm7
+; X86-SSE2-NEXT: paddb %xmm5, %xmm7
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm5
; X86-SSE2-NEXT: psrlw $1, %xmm5
; X86-SSE2-NEXT: pand %xmm4, %xmm5
-; X86-SSE2-NEXT: psubb %xmm5, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
+; X86-SSE2-NEXT: psubb %xmm5, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm4
; X86-SSE2-NEXT: pand %xmm3, %xmm4
-; X86-SSE2-NEXT: psrlw $2, %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: paddb %xmm4, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X86-SSE2-NEXT: psrlw $2, %xmm2
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: paddb %xmm4, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
; X86-SSE2-NEXT: psrlw $4, %xmm3
-; X86-SSE2-NEXT: paddb %xmm1, %xmm3
-; X86-SSE2-NEXT: pand %xmm2, %xmm3
-; X86-SSE2-NEXT: paddb %xmm0, %xmm3
-; X86-SSE2-NEXT: paddb %xmm6, %xmm3
-; X86-SSE2-NEXT: pxor %xmm0, %xmm0
-; X86-SSE2-NEXT: psadbw %xmm3, %xmm0
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm0, %xmm1
-; X86-SSE2-NEXT: movd %xmm1, %eax
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X86-SSE2-NEXT: movd %xmm0, %edx
+; X86-SSE2-NEXT: paddb %xmm2, %xmm3
+; X86-SSE2-NEXT: pand %xmm1, %xmm7
+; X86-SSE2-NEXT: pand %xmm1, %xmm3
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm7
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm7, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm3, %xmm6
+; X86-SSE2-NEXT: packuswb %xmm0, %xmm6
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm6
+; X86-SSE2-NEXT: movd %xmm6, %eax
+; X86-SSE2-NEXT: xorl %edx, %edx
; X86-SSE2-NEXT: movl %ebp, %esp
; X86-SSE2-NEXT: popl %ebp
; X86-SSE2-NEXT: retl
;
; X64-SSE2-LABEL: reduce_ctpop_v8i64:
; X64-SSE2: # %bb.0:
-; X64-SSE2-NEXT: movdqa %xmm2, %xmm4
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm4
; X64-SSE2-NEXT: psrlw $1, %xmm4
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; X64-SSE2-NEXT: pand %xmm5, %xmm4
-; X64-SSE2-NEXT: psubb %xmm4, %xmm2
+; X64-SSE2-NEXT: psubb %xmm4, %xmm1
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; X64-SSE2-NEXT: movdqa %xmm2, %xmm6
-; X64-SSE2-NEXT: pand %xmm4, %xmm6
-; X64-SSE2-NEXT: psrlw $2, %xmm2
-; X64-SSE2-NEXT: pand %xmm4, %xmm2
-; X64-SSE2-NEXT: paddb %xmm6, %xmm2
-; X64-SSE2-NEXT: movdqa %xmm2, %xmm7
-; X64-SSE2-NEXT: psrlw $4, %xmm7
-; X64-SSE2-NEXT: paddb %xmm2, %xmm7
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-SSE2-NEXT: pand %xmm2, %xmm7
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm6
-; X64-SSE2-NEXT: psrlw $1, %xmm6
-; X64-SSE2-NEXT: pand %xmm5, %xmm6
-; X64-SSE2-NEXT: psubb %xmm6, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm6
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm6
; X64-SSE2-NEXT: pand %xmm4, %xmm6
+; X64-SSE2-NEXT: psrlw $2, %xmm1
+; X64-SSE2-NEXT: pand %xmm4, %xmm1
+; X64-SSE2-NEXT: paddb %xmm6, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm8
+; X64-SSE2-NEXT: psrlw $4, %xmm8
+; X64-SSE2-NEXT: paddb %xmm1, %xmm8
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-SSE2-NEXT: pand %xmm6, %xmm8
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm8
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm7
+; X64-SSE2-NEXT: psrlw $1, %xmm7
+; X64-SSE2-NEXT: pand %xmm5, %xmm7
+; X64-SSE2-NEXT: psubb %xmm7, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm7
+; X64-SSE2-NEXT: pand %xmm4, %xmm7
; X64-SSE2-NEXT: psrlw $2, %xmm0
; X64-SSE2-NEXT: pand %xmm4, %xmm0
-; X64-SSE2-NEXT: paddb %xmm6, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm6
-; X64-SSE2-NEXT: psrlw $4, %xmm6
-; X64-SSE2-NEXT: paddb %xmm0, %xmm6
-; X64-SSE2-NEXT: pand %xmm2, %xmm6
-; X64-SSE2-NEXT: paddb %xmm7, %xmm6
+; X64-SSE2-NEXT: paddb %xmm7, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm7
+; X64-SSE2-NEXT: psrlw $4, %xmm7
+; X64-SSE2-NEXT: paddb %xmm0, %xmm7
+; X64-SSE2-NEXT: pand %xmm6, %xmm7
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm7
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm7
; X64-SSE2-NEXT: movdqa %xmm3, %xmm0
; X64-SSE2-NEXT: psrlw $1, %xmm0
; X64-SSE2-NEXT: pand %xmm5, %xmm0
@@ -1353,27 +1412,27 @@ define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) nounwind {
; X64-SSE2-NEXT: movdqa %xmm3, %xmm0
; X64-SSE2-NEXT: psrlw $4, %xmm0
; X64-SSE2-NEXT: paddb %xmm3, %xmm0
-; X64-SSE2-NEXT: pand %xmm2, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE2-NEXT: pand %xmm6, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
; X64-SSE2-NEXT: psrlw $1, %xmm3
; X64-SSE2-NEXT: pand %xmm5, %xmm3
-; X64-SSE2-NEXT: psubb %xmm3, %xmm1
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE2-NEXT: psubb %xmm3, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
; X64-SSE2-NEXT: pand %xmm4, %xmm3
-; X64-SSE2-NEXT: psrlw $2, %xmm1
-; X64-SSE2-NEXT: pand %xmm4, %xmm1
-; X64-SSE2-NEXT: paddb %xmm3, %xmm1
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE2-NEXT: psrlw $2, %xmm2
+; X64-SSE2-NEXT: pand %xmm4, %xmm2
+; X64-SSE2-NEXT: paddb %xmm3, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
; X64-SSE2-NEXT: psrlw $4, %xmm3
-; X64-SSE2-NEXT: paddb %xmm1, %xmm3
-; X64-SSE2-NEXT: pand %xmm2, %xmm3
-; X64-SSE2-NEXT: paddb %xmm0, %xmm3
-; X64-SSE2-NEXT: paddb %xmm6, %xmm3
-; X64-SSE2-NEXT: pxor %xmm0, %xmm0
-; X64-SSE2-NEXT: psadbw %xmm3, %xmm0
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm0, %xmm1
-; X64-SSE2-NEXT: movq %xmm1, %rax
+; X64-SSE2-NEXT: paddb %xmm2, %xmm3
+; X64-SSE2-NEXT: pand %xmm6, %xmm3
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm0, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm3, %xmm7
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm7
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm7
+; X64-SSE2-NEXT: movq %xmm7, %rax
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: reduce_ctpop_v8i64:
@@ -1384,49 +1443,52 @@ define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) nounwind {
; X86-SSE4-NEXT: subl $16, %esp
; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm5
; X86-SSE4-NEXT: movdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm6
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm6
; X86-SSE4-NEXT: pand %xmm4, %xmm6
; X86-SSE4-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
; X86-SSE4-NEXT: pshufb %xmm6, %xmm7
-; X86-SSE4-NEXT: psrlw $4, %xmm2
-; X86-SSE4-NEXT: pand %xmm4, %xmm2
+; X86-SSE4-NEXT: psrlw $4, %xmm1
+; X86-SSE4-NEXT: pand %xmm4, %xmm1
; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm6
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm6
; X86-SSE4-NEXT: paddb %xmm7, %xmm6
-; X86-SSE4-NEXT: movdqa %xmm0, %xmm2
-; X86-SSE4-NEXT: pand %xmm4, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE4-NEXT: pand %xmm4, %xmm1
; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm7
; X86-SSE4-NEXT: psrlw $4, %xmm0
; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm2
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm2
-; X86-SSE4-NEXT: paddb %xmm7, %xmm2
-; X86-SSE4-NEXT: paddb %xmm6, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm5, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm6
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm1
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm1
+; X86-SSE4-NEXT: pxor %xmm0, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm6
+; X86-SSE4-NEXT: paddb %xmm7, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE4-NEXT: packusdw %xmm6, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm5, %xmm6
+; X86-SSE4-NEXT: pand %xmm4, %xmm6
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm6, %xmm7
; X86-SSE4-NEXT: psrlw $4, %xmm5
; X86-SSE4-NEXT: pand %xmm4, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
-; X86-SSE4-NEXT: pshufb %xmm5, %xmm0
-; X86-SSE4-NEXT: paddb %xmm6, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm5
-; X86-SSE4-NEXT: pand %xmm4, %xmm5
; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
; X86-SSE4-NEXT: pshufb %xmm5, %xmm6
-; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm3
-; X86-SSE4-NEXT: paddb %xmm6, %xmm3
-; X86-SSE4-NEXT: paddb %xmm0, %xmm3
-; X86-SSE4-NEXT: paddb %xmm2, %xmm3
-; X86-SSE4-NEXT: pxor %xmm0, %xmm0
-; X86-SSE4-NEXT: psadbw %xmm3, %xmm0
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm0, %xmm1
+; X86-SSE4-NEXT: paddb %xmm7, %xmm6
+; X86-SSE4-NEXT: movdqa %xmm2, %xmm5
+; X86-SSE4-NEXT: pand %xmm4, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm5, %xmm7
+; X86-SSE4-NEXT: psrlw $4, %xmm2
+; X86-SSE4-NEXT: pand %xmm4, %xmm2
+; X86-SSE4-NEXT: pshufb %xmm2, %xmm3
+; X86-SSE4-NEXT: paddb %xmm7, %xmm3
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm6
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm3
+; X86-SSE4-NEXT: packusdw %xmm6, %xmm3
+; X86-SSE4-NEXT: packusdw %xmm3, %xmm1
+; X86-SSE4-NEXT: packuswb %xmm0, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
; X86-SSE4-NEXT: movd %xmm1, %eax
; X86-SSE4-NEXT: pextrd $1, %xmm1, %edx
; X86-SSE4-NEXT: movl %ebp, %esp
@@ -1436,89 +1498,95 @@ define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) nounwind {
; X64-SSE4-LABEL: reduce_ctpop_v8i64:
; X64-SSE4: # %bb.0:
; X64-SSE4-NEXT: movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-SSE4-NEXT: movdqa %xmm2, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm6
; X64-SSE4-NEXT: pand %xmm5, %xmm6
; X64-SSE4-NEXT: movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
; X64-SSE4-NEXT: movdqa %xmm4, %xmm7
; X64-SSE4-NEXT: pshufb %xmm6, %xmm7
-; X64-SSE4-NEXT: psrlw $4, %xmm2
-; X64-SSE4-NEXT: pand %xmm5, %xmm2
-; X64-SSE4-NEXT: movdqa %xmm4, %xmm6
-; X64-SSE4-NEXT: pshufb %xmm2, %xmm6
-; X64-SSE4-NEXT: paddb %xmm7, %xmm6
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE4-NEXT: pand %xmm5, %xmm2
+; X64-SSE4-NEXT: psrlw $4, %xmm1
+; X64-SSE4-NEXT: pand %xmm5, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm4, %xmm8
+; X64-SSE4-NEXT: pshufb %xmm1, %xmm8
+; X64-SSE4-NEXT: paddb %xmm7, %xmm8
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm8
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm6
+; X64-SSE4-NEXT: pand %xmm5, %xmm6
; X64-SSE4-NEXT: movdqa %xmm4, %xmm7
-; X64-SSE4-NEXT: pshufb %xmm2, %xmm7
+; X64-SSE4-NEXT: pshufb %xmm6, %xmm7
; X64-SSE4-NEXT: psrlw $4, %xmm0
; X64-SSE4-NEXT: pand %xmm5, %xmm0
-; X64-SSE4-NEXT: movdqa %xmm4, %xmm2
-; X64-SSE4-NEXT: pshufb %xmm0, %xmm2
-; X64-SSE4-NEXT: paddb %xmm7, %xmm2
-; X64-SSE4-NEXT: paddb %xmm6, %xmm2
-; X64-SSE4-NEXT: movdqa %xmm3, %xmm0
-; X64-SSE4-NEXT: pand %xmm5, %xmm0
; X64-SSE4-NEXT: movdqa %xmm4, %xmm6
; X64-SSE4-NEXT: pshufb %xmm0, %xmm6
+; X64-SSE4-NEXT: paddb %xmm7, %xmm6
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm6
+; X64-SSE4-NEXT: packusdw %xmm8, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm3, %xmm0
+; X64-SSE4-NEXT: pand %xmm5, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm4, %xmm7
+; X64-SSE4-NEXT: pshufb %xmm0, %xmm7
; X64-SSE4-NEXT: psrlw $4, %xmm3
; X64-SSE4-NEXT: pand %xmm5, %xmm3
; X64-SSE4-NEXT: movdqa %xmm4, %xmm0
; X64-SSE4-NEXT: pshufb %xmm3, %xmm0
-; X64-SSE4-NEXT: paddb %xmm6, %xmm0
-; X64-SSE4-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE4-NEXT: paddb %xmm7, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm2, %xmm3
; X64-SSE4-NEXT: pand %xmm5, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm4, %xmm6
-; X64-SSE4-NEXT: pshufb %xmm3, %xmm6
-; X64-SSE4-NEXT: psrlw $4, %xmm1
-; X64-SSE4-NEXT: pand %xmm5, %xmm1
-; X64-SSE4-NEXT: pshufb %xmm1, %xmm4
-; X64-SSE4-NEXT: paddb %xmm6, %xmm4
-; X64-SSE4-NEXT: paddb %xmm0, %xmm4
-; X64-SSE4-NEXT: paddb %xmm2, %xmm4
-; X64-SSE4-NEXT: pxor %xmm0, %xmm0
-; X64-SSE4-NEXT: psadbw %xmm4, %xmm0
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm0, %xmm1
-; X64-SSE4-NEXT: movq %xmm1, %rax
+; X64-SSE4-NEXT: movdqa %xmm4, %xmm7
+; X64-SSE4-NEXT: pshufb %xmm3, %xmm7
+; X64-SSE4-NEXT: psrlw $4, %xmm2
+; X64-SSE4-NEXT: pand %xmm5, %xmm2
+; X64-SSE4-NEXT: pshufb %xmm2, %xmm4
+; X64-SSE4-NEXT: paddb %xmm7, %xmm4
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm4
+; X64-SSE4-NEXT: packusdw %xmm0, %xmm4
+; X64-SSE4-NEXT: packusdw %xmm4, %xmm6
+; X64-SSE4-NEXT: packuswb %xmm1, %xmm6
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm6
+; X64-SSE4-NEXT: movq %xmm6, %rax
; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: reduce_ctpop_v8i64:
; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm4
+; X86-AVX1-NEXT: vpand %xmm2, %xmm4, %xmm5
; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X86-AVX1-NEXT: vpshufb %xmm4, %xmm3, %xmm4
-; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm5
-; X86-AVX1-NEXT: vpand %xmm2, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm5
-; X86-AVX1-NEXT: vpaddb %xmm4, %xmm5, %xmm4
-; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm5
; X86-AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm5
-; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm6
-; X86-AVX1-NEXT: vpand %xmm2, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpand %xmm2, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpshufb %xmm4, %xmm3, %xmm4
+; X86-AVX1-NEXT: vpaddb %xmm5, %xmm4, %xmm5
+; X86-AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm6
; X86-AVX1-NEXT: vpshufb %xmm6, %xmm3, %xmm6
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm6, %xmm5
-; X86-AVX1-NEXT: vpaddb %xmm4, %xmm5, %xmm4
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm5
-; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm3, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm1, %xmm1
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm5
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-NEXT: vpand %xmm2, %xmm5, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm3, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpand %xmm2, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm5
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm3, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm3, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm5, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: vpextrd $1, %xmm0, %edx
; X86-AVX1-NEXT: vzeroupper
@@ -1526,41 +1594,44 @@ define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) nounwind {
;
; X64-AVX1-LABEL: reduce_ctpop_v8i64:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm3
-; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X64-AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
-; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm5
-; X64-AVX1-NEXT: vpand %xmm2, %xmm5, %xmm5
-; X64-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X64-AVX1-NEXT: vpaddb %xmm3, %xmm5, %xmm3
-; X64-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm5
-; X64-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm6
-; X64-AVX1-NEXT: vpand %xmm2, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
-; X64-AVX1-NEXT: vpaddb %xmm5, %xmm6, %xmm5
-; X64-AVX1-NEXT: vpaddb %xmm3, %xmm5, %xmm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X64-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm5
-; X64-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm5, %xmm1, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; X64-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm5
-; X64-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm4
+; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm4
+; X64-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
+; X64-AVX1-NEXT: vpaddb %xmm4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm6
+; X64-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm0, %xmm3, %xmm0
-; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
+; X64-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
+; X64-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm6
+; X64-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
+; X64-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
+; X64-AVX1-NEXT: vpaddb %xmm6, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm6
+; X64-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
+; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
+; X64-AVX1-NEXT: vpaddb %xmm6, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
; X64-AVX1-NEXT: vzeroupper
; X64-AVX1-NEXT: retq
@@ -1576,19 +1647,23 @@ define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) nounwind {
; X86-AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
; X86-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
; X86-AVX2-NEXT: vpaddb %ymm3, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
-; X86-AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3
+; X86-AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X86-AVX2-NEXT: vpsadbw %ymm3, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm5
+; X86-AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm5
; X86-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
; X86-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
-; X86-AVX2-NEXT: vpaddb %ymm3, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpsadbw %ymm3, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm3, %ymm0, %ymm0
; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: vpextrd $1, %xmm0, %edx
; X86-AVX2-NEXT: vzeroupper
@@ -1605,19 +1680,23 @@ define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) nounwind {
; X64-AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
; X64-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
; X64-AVX2-NEXT: vpaddb %ymm3, %ymm1, %ymm1
-; X64-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
-; X64-AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3
+; X64-AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X64-AVX2-NEXT: vpsadbw %ymm3, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm5
+; X64-AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm5
; X64-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
; X64-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
; X64-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
-; X64-AVX2-NEXT: vpaddb %ymm3, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpsadbw %ymm3, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm3, %ymm0, %ymm0
; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovq %xmm0, %rax
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
@@ -1663,28 +1742,28 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; X86-SSE2-NEXT: movl %esp, %ebp
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm3
; X86-SSE2-NEXT: psrlw $1, %xmm3
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; X86-SSE2-NEXT: pand %xmm4, %xmm3
-; X86-SSE2-NEXT: psubb %xmm3, %xmm2
+; X86-SSE2-NEXT: psubb %xmm3, %xmm1
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm5
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm5
; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: psrlw $2, %xmm2
-; X86-SSE2-NEXT: pand %xmm3, %xmm2
-; X86-SSE2-NEXT: paddb %xmm5, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm7
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm3, %xmm1
+; X86-SSE2-NEXT: paddb %xmm5, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm7
; X86-SSE2-NEXT: psrlw $4, %xmm7
-; X86-SSE2-NEXT: paddb %xmm2, %xmm7
+; X86-SSE2-NEXT: paddb %xmm1, %xmm7
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
; X86-SSE2-NEXT: pand %xmm5, %xmm7
-; X86-SSE2-NEXT: pxor %xmm2, %xmm2
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
; X86-SSE2-NEXT: movdqa %xmm7, %xmm6
-; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; X86-SSE2-NEXT: psadbw %xmm2, %xmm6
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
-; X86-SSE2-NEXT: psadbw %xmm2, %xmm7
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm1[2],xmm6[3],xmm1[3]
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm6
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm7
; X86-SSE2-NEXT: packuswb %xmm6, %xmm7
; X86-SSE2-NEXT: movdqa %xmm0, %xmm6
; X86-SSE2-NEXT: psrlw $1, %xmm6
@@ -1700,13 +1779,13 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; X86-SSE2-NEXT: paddb %xmm0, %xmm6
; X86-SSE2-NEXT: pand %xmm5, %xmm6
; X86-SSE2-NEXT: movdqa %xmm6, %xmm0
-; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; X86-SSE2-NEXT: psadbw %xmm2, %xmm0
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
-; X86-SSE2-NEXT: psadbw %xmm2, %xmm6
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm6
; X86-SSE2-NEXT: packuswb %xmm0, %xmm6
; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm0
-; X86-SSE2-NEXT: paddd %xmm7, %xmm6
+; X86-SSE2-NEXT: packuswb %xmm7, %xmm6
; X86-SSE2-NEXT: movdqa %xmm0, %xmm7
; X86-SSE2-NEXT: psrlw $1, %xmm7
; X86-SSE2-NEXT: pand %xmm4, %xmm7
@@ -1721,65 +1800,64 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; X86-SSE2-NEXT: paddb %xmm0, %xmm7
; X86-SSE2-NEXT: pand %xmm5, %xmm7
; X86-SSE2-NEXT: movdqa %xmm7, %xmm0
-; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; X86-SSE2-NEXT: psadbw %xmm2, %xmm0
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
-; X86-SSE2-NEXT: psadbw %xmm2, %xmm7
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm7
; X86-SSE2-NEXT: packuswb %xmm0, %xmm7
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
; X86-SSE2-NEXT: psrlw $1, %xmm0
; X86-SSE2-NEXT: pand %xmm4, %xmm0
-; X86-SSE2-NEXT: psubb %xmm0, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: psubb %xmm0, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: psrlw $2, %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: paddb %xmm0, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm3
-; X86-SSE2-NEXT: psrlw $4, %xmm3
-; X86-SSE2-NEXT: paddb %xmm1, %xmm3
-; X86-SSE2-NEXT: pand %xmm5, %xmm3
-; X86-SSE2-NEXT: movdqa %xmm3, %xmm0
-; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; X86-SSE2-NEXT: psadbw %xmm2, %xmm0
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X86-SSE2-NEXT: psadbw %xmm2, %xmm3
-; X86-SSE2-NEXT: packuswb %xmm0, %xmm3
-; X86-SSE2-NEXT: paddd %xmm7, %xmm3
-; X86-SSE2-NEXT: paddd %xmm6, %xmm3
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
-; X86-SSE2-NEXT: paddd %xmm3, %xmm0
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-SSE2-NEXT: paddd %xmm0, %xmm1
-; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: psrlw $2, %xmm2
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: paddb %xmm0, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: psrlw $4, %xmm0
+; X86-SSE2-NEXT: paddb %xmm2, %xmm0
+; X86-SSE2-NEXT: pand %xmm5, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm2
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm7, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm0, %xmm6
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm6
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm6[2,3,2,3]
+; X86-SSE2-NEXT: paddq %xmm6, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
; X86-SSE2-NEXT: movl %ebp, %esp
; X86-SSE2-NEXT: popl %ebp
; X86-SSE2-NEXT: retl
;
; X64-SSE2-LABEL: reduce_ctpop_v16i32:
; X64-SSE2: # %bb.0:
-; X64-SSE2-NEXT: movdqa %xmm2, %xmm4
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm4
; X64-SSE2-NEXT: psrlw $1, %xmm4
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; X64-SSE2-NEXT: pand %xmm5, %xmm4
-; X64-SSE2-NEXT: psubb %xmm4, %xmm2
+; X64-SSE2-NEXT: psubb %xmm4, %xmm1
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; X64-SSE2-NEXT: movdqa %xmm2, %xmm6
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm6
; X64-SSE2-NEXT: pand %xmm4, %xmm6
-; X64-SSE2-NEXT: psrlw $2, %xmm2
-; X64-SSE2-NEXT: pand %xmm4, %xmm2
-; X64-SSE2-NEXT: paddb %xmm6, %xmm2
-; X64-SSE2-NEXT: movdqa %xmm2, %xmm8
+; X64-SSE2-NEXT: psrlw $2, %xmm1
+; X64-SSE2-NEXT: pand %xmm4, %xmm1
+; X64-SSE2-NEXT: paddb %xmm6, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm8
; X64-SSE2-NEXT: psrlw $4, %xmm8
-; X64-SSE2-NEXT: paddb %xmm2, %xmm8
+; X64-SSE2-NEXT: paddb %xmm1, %xmm8
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
; X64-SSE2-NEXT: pand %xmm6, %xmm8
-; X64-SSE2-NEXT: pxor %xmm2, %xmm2
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
; X64-SSE2-NEXT: movdqa %xmm8, %xmm7
-; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm2[2],xmm7[3],xmm2[3]
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm7
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm2[0],xmm8[1],xmm2[1]
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm8
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm1[2],xmm7[3],xmm1[3]
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm7
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm1[0],xmm8[1],xmm1[1]
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm8
; X64-SSE2-NEXT: packuswb %xmm7, %xmm8
; X64-SSE2-NEXT: movdqa %xmm0, %xmm7
; X64-SSE2-NEXT: psrlw $1, %xmm7
@@ -1795,12 +1873,12 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; X64-SSE2-NEXT: paddb %xmm0, %xmm7
; X64-SSE2-NEXT: pand %xmm6, %xmm7
; X64-SSE2-NEXT: movdqa %xmm7, %xmm0
-; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm0
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm7
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm7
; X64-SSE2-NEXT: packuswb %xmm0, %xmm7
-; X64-SSE2-NEXT: paddd %xmm8, %xmm7
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm7
; X64-SSE2-NEXT: movdqa %xmm3, %xmm0
; X64-SSE2-NEXT: psrlw $1, %xmm0
; X64-SSE2-NEXT: pand %xmm5, %xmm0
@@ -1815,37 +1893,37 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; X64-SSE2-NEXT: paddb %xmm3, %xmm0
; X64-SSE2-NEXT: pand %xmm6, %xmm0
; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm3
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm0
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm3
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
; X64-SSE2-NEXT: packuswb %xmm3, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
; X64-SSE2-NEXT: psrlw $1, %xmm3
; X64-SSE2-NEXT: pand %xmm5, %xmm3
-; X64-SSE2-NEXT: psubb %xmm3, %xmm1
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE2-NEXT: psubb %xmm3, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
; X64-SSE2-NEXT: pand %xmm4, %xmm3
-; X64-SSE2-NEXT: psrlw $2, %xmm1
-; X64-SSE2-NEXT: pand %xmm4, %xmm1
-; X64-SSE2-NEXT: paddb %xmm3, %xmm1
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE2-NEXT: psrlw $2, %xmm2
+; X64-SSE2-NEXT: pand %xmm4, %xmm2
+; X64-SSE2-NEXT: paddb %xmm3, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
; X64-SSE2-NEXT: psrlw $4, %xmm3
-; X64-SSE2-NEXT: paddb %xmm1, %xmm3
+; X64-SSE2-NEXT: paddb %xmm2, %xmm3
; X64-SSE2-NEXT: pand %xmm6, %xmm3
-; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
-; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm1
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm3
-; X64-SSE2-NEXT: packuswb %xmm1, %xmm3
-; X64-SSE2-NEXT: paddd %xmm0, %xmm3
-; X64-SSE2-NEXT: paddd %xmm7, %xmm3
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
-; X64-SSE2-NEXT: paddd %xmm3, %xmm0
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-SSE2-NEXT: paddd %xmm0, %xmm1
-; X64-SSE2-NEXT: movd %xmm1, %eax
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm2
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm2
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm2, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm0, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm3, %xmm7
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm7
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm7[2,3,2,3]
+; X64-SSE2-NEXT: paddq %xmm7, %xmm0
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: # kill: def $eax killed $eax killed $rax
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: reduce_ctpop_v16i32:
@@ -1855,72 +1933,73 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
; X86-SSE4-NEXT: movdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm5
; X86-SSE4-NEXT: pand %xmm4, %xmm5
; X86-SSE4-NEXT: movdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
; X86-SSE4-NEXT: movdqa %xmm6, %xmm3
; X86-SSE4-NEXT: pshufb %xmm5, %xmm3
-; X86-SSE4-NEXT: psrlw $4, %xmm2
-; X86-SSE4-NEXT: pand %xmm4, %xmm2
+; X86-SSE4-NEXT: psrlw $4, %xmm1
+; X86-SSE4-NEXT: pand %xmm4, %xmm1
; X86-SSE4-NEXT: movdqa %xmm6, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm7
; X86-SSE4-NEXT: paddb %xmm3, %xmm7
-; X86-SSE4-NEXT: pxor %xmm2, %xmm2
+; X86-SSE4-NEXT: pxor %xmm4, %xmm4
; X86-SSE4-NEXT: pmovzxdq {{.*#+}} xmm5 = xmm7[0],zero,xmm7[1],zero
-; X86-SSE4-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm2[2],xmm7[3],xmm2[3]
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm7
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm5
+; X86-SSE4-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm4[2],xmm7[3],xmm4[3]
+; X86-SSE4-NEXT: psadbw %xmm4, %xmm7
+; X86-SSE4-NEXT: psadbw %xmm4, %xmm5
; X86-SSE4-NEXT: packuswb %xmm7, %xmm5
; X86-SSE4-NEXT: movdqa %xmm0, %xmm3
-; X86-SSE4-NEXT: pand %xmm4, %xmm3
+; X86-SSE4-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE4-NEXT: pand %xmm1, %xmm3
; X86-SSE4-NEXT: movdqa %xmm6, %xmm7
; X86-SSE4-NEXT: pshufb %xmm3, %xmm7
; X86-SSE4-NEXT: psrlw $4, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
+; X86-SSE4-NEXT: pand %xmm1, %xmm0
; X86-SSE4-NEXT: movdqa %xmm6, %xmm3
; X86-SSE4-NEXT: pshufb %xmm0, %xmm3
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm0
+; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm1
; X86-SSE4-NEXT: paddb %xmm7, %xmm3
-; X86-SSE4-NEXT: pmovzxdq {{.*#+}} xmm7 = xmm3[0],zero,xmm3[1],zero
-; X86-SSE4-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm3
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm7
-; X86-SSE4-NEXT: packuswb %xmm3, %xmm7
-; X86-SSE4-NEXT: paddd %xmm5, %xmm7
-; X86-SSE4-NEXT: movdqa %xmm0, %xmm3
-; X86-SSE4-NEXT: pand %xmm4, %xmm3
+; X86-SSE4-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm3[0],zero,xmm3[1],zero
+; X86-SSE4-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; X86-SSE4-NEXT: psadbw %xmm4, %xmm3
+; X86-SSE4-NEXT: psadbw %xmm4, %xmm0
+; X86-SSE4-NEXT: packuswb %xmm3, %xmm0
+; X86-SSE4-NEXT: packusdw %xmm5, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm3
+; X86-SSE4-NEXT: movdqa {{.*#+}} xmm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE4-NEXT: pand %xmm7, %xmm3
; X86-SSE4-NEXT: movdqa %xmm6, %xmm5
; X86-SSE4-NEXT: pshufb %xmm3, %xmm5
-; X86-SSE4-NEXT: psrlw $4, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
+; X86-SSE4-NEXT: psrlw $4, %xmm1
+; X86-SSE4-NEXT: pand %xmm7, %xmm1
; X86-SSE4-NEXT: movdqa %xmm6, %xmm3
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm3
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm3
; X86-SSE4-NEXT: paddb %xmm5, %xmm3
; X86-SSE4-NEXT: pmovzxdq {{.*#+}} xmm5 = xmm3[0],zero,xmm3[1],zero
-; X86-SSE4-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm3
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm5
+; X86-SSE4-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; X86-SSE4-NEXT: psadbw %xmm4, %xmm3
+; X86-SSE4-NEXT: psadbw %xmm4, %xmm5
; X86-SSE4-NEXT: packuswb %xmm3, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm2, %xmm1
+; X86-SSE4-NEXT: pand %xmm7, %xmm1
; X86-SSE4-NEXT: movdqa %xmm6, %xmm3
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm3
-; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm6
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm3
+; X86-SSE4-NEXT: psrlw $4, %xmm2
+; X86-SSE4-NEXT: pand %xmm7, %xmm2
+; X86-SSE4-NEXT: pshufb %xmm2, %xmm6
; X86-SSE4-NEXT: paddb %xmm3, %xmm6
-; X86-SSE4-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm6[0],zero,xmm6[1],zero
-; X86-SSE4-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm6
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm0
-; X86-SSE4-NEXT: packuswb %xmm6, %xmm0
-; X86-SSE4-NEXT: paddd %xmm5, %xmm0
-; X86-SSE4-NEXT: paddd %xmm7, %xmm0
+; X86-SSE4-NEXT: pmovzxdq {{.*#+}} xmm1 = xmm6[0],zero,xmm6[1],zero
+; X86-SSE4-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; X86-SSE4-NEXT: psadbw %xmm4, %xmm6
+; X86-SSE4-NEXT: psadbw %xmm4, %xmm1
+; X86-SSE4-NEXT: packuswb %xmm6, %xmm1
+; X86-SSE4-NEXT: packusdw %xmm5, %xmm1
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm4, %xmm0
; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE4-NEXT: paddd %xmm0, %xmm1
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X86-SSE4-NEXT: paddd %xmm1, %xmm0
-; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: paddq %xmm0, %xmm1
+; X86-SSE4-NEXT: movd %xmm1, %eax
; X86-SSE4-NEXT: movl %ebp, %esp
; X86-SSE4-NEXT: popl %ebp
; X86-SSE4-NEXT: retl
@@ -1928,21 +2007,21 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; X64-SSE4-LABEL: reduce_ctpop_v16i32:
; X64-SSE4: # %bb.0:
; X64-SSE4-NEXT: movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-SSE4-NEXT: movdqa %xmm2, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm6
; X64-SSE4-NEXT: pand %xmm5, %xmm6
; X64-SSE4-NEXT: movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
; X64-SSE4-NEXT: movdqa %xmm4, %xmm7
; X64-SSE4-NEXT: pshufb %xmm6, %xmm7
-; X64-SSE4-NEXT: psrlw $4, %xmm2
-; X64-SSE4-NEXT: pand %xmm5, %xmm2
+; X64-SSE4-NEXT: psrlw $4, %xmm1
+; X64-SSE4-NEXT: pand %xmm5, %xmm1
; X64-SSE4-NEXT: movdqa %xmm4, %xmm6
-; X64-SSE4-NEXT: pshufb %xmm2, %xmm6
+; X64-SSE4-NEXT: pshufb %xmm1, %xmm6
; X64-SSE4-NEXT: paddb %xmm7, %xmm6
-; X64-SSE4-NEXT: pxor %xmm2, %xmm2
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
; X64-SSE4-NEXT: pmovzxdq {{.*#+}} xmm7 = xmm6[0],zero,xmm6[1],zero
-; X64-SSE4-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; X64-SSE4-NEXT: psadbw %xmm2, %xmm6
-; X64-SSE4-NEXT: psadbw %xmm2, %xmm7
+; X64-SSE4-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm1[2],xmm6[3],xmm1[3]
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm6
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm7
; X64-SSE4-NEXT: packuswb %xmm6, %xmm7
; X64-SSE4-NEXT: movdqa %xmm0, %xmm6
; X64-SSE4-NEXT: pand %xmm5, %xmm6
@@ -1954,11 +2033,11 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; X64-SSE4-NEXT: pshufb %xmm0, %xmm6
; X64-SSE4-NEXT: paddb %xmm8, %xmm6
; X64-SSE4-NEXT: pmovzxdq {{.*#+}} xmm0 = xmm6[0],zero,xmm6[1],zero
-; X64-SSE4-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; X64-SSE4-NEXT: psadbw %xmm2, %xmm6
-; X64-SSE4-NEXT: psadbw %xmm2, %xmm0
+; X64-SSE4-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm1[2],xmm6[3],xmm1[3]
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm6
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
; X64-SSE4-NEXT: packuswb %xmm6, %xmm0
-; X64-SSE4-NEXT: paddd %xmm7, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm7, %xmm0
; X64-SSE4-NEXT: movdqa %xmm3, %xmm6
; X64-SSE4-NEXT: pand %xmm5, %xmm6
; X64-SSE4-NEXT: movdqa %xmm4, %xmm7
@@ -1969,136 +2048,239 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; X64-SSE4-NEXT: pshufb %xmm3, %xmm6
; X64-SSE4-NEXT: paddb %xmm7, %xmm6
; X64-SSE4-NEXT: pmovzxdq {{.*#+}} xmm3 = xmm6[0],zero,xmm6[1],zero
-; X64-SSE4-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; X64-SSE4-NEXT: psadbw %xmm2, %xmm6
-; X64-SSE4-NEXT: psadbw %xmm2, %xmm3
+; X64-SSE4-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm1[2],xmm6[3],xmm1[3]
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm6
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm3
; X64-SSE4-NEXT: packuswb %xmm6, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm1, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm2, %xmm6
; X64-SSE4-NEXT: pand %xmm5, %xmm6
; X64-SSE4-NEXT: movdqa %xmm4, %xmm7
; X64-SSE4-NEXT: pshufb %xmm6, %xmm7
-; X64-SSE4-NEXT: psrlw $4, %xmm1
-; X64-SSE4-NEXT: pand %xmm5, %xmm1
-; X64-SSE4-NEXT: pshufb %xmm1, %xmm4
+; X64-SSE4-NEXT: psrlw $4, %xmm2
+; X64-SSE4-NEXT: pand %xmm5, %xmm2
+; X64-SSE4-NEXT: pshufb %xmm2, %xmm4
; X64-SSE4-NEXT: paddb %xmm7, %xmm4
-; X64-SSE4-NEXT: pmovzxdq {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero
-; X64-SSE4-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
-; X64-SSE4-NEXT: psadbw %xmm2, %xmm4
-; X64-SSE4-NEXT: psadbw %xmm2, %xmm1
-; X64-SSE4-NEXT: packuswb %xmm4, %xmm1
-; X64-SSE4-NEXT: paddd %xmm3, %xmm1
-; X64-SSE4-NEXT: paddd %xmm0, %xmm1
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X64-SSE4-NEXT: paddd %xmm1, %xmm0
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-SSE4-NEXT: paddd %xmm0, %xmm1
-; X64-SSE4-NEXT: movd %xmm1, %eax
+; X64-SSE4-NEXT: pmovzxdq {{.*#+}} xmm2 = xmm4[0],zero,xmm4[1],zero
+; X64-SSE4-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm4
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm2
+; X64-SSE4-NEXT: packuswb %xmm4, %xmm2
+; X64-SSE4-NEXT: packusdw %xmm3, %xmm2
+; X64-SSE4-NEXT: packuswb %xmm2, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm0, %xmm1
+; X64-SSE4-NEXT: movq %xmm1, %rax
+; X64-SSE4-NEXT: # kill: def $eax killed $eax killed $rax
; X64-SSE4-NEXT: retq
;
-; AVX1-LABEL: reduce_ctpop_v16i32:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm4
-; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX1-NEXT: vpshufb %xmm4, %xmm3, %xmm4
-; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm5
-; AVX1-NEXT: vpand %xmm2, %xmm5, %xmm5
-; AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm5
-; AVX1-NEXT: vpaddb %xmm4, %xmm5, %xmm5
-; AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm5[2],xmm4[2],xmm5[3],xmm4[3]
-; AVX1-NEXT: vpsadbw %xmm4, %xmm6, %xmm6
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero
-; AVX1-NEXT: vpsadbw %xmm4, %xmm5, %xmm5
-; AVX1-NEXT: vpackuswb %xmm6, %xmm5, %xmm5
-; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm6
-; AVX1-NEXT: vpshufb %xmm6, %xmm3, %xmm6
-; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm7
-; AVX1-NEXT: vpand %xmm2, %xmm7, %xmm7
-; AVX1-NEXT: vpshufb %xmm7, %xmm3, %xmm7
-; AVX1-NEXT: vpaddb %xmm6, %xmm7, %xmm6
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm7 = xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX1-NEXT: vpsadbw %xmm4, %xmm7, %xmm7
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm6 = xmm6[0],zero,xmm6[1],zero
-; AVX1-NEXT: vpsadbw %xmm4, %xmm6, %xmm6
-; AVX1-NEXT: vpackuswb %xmm7, %xmm6, %xmm6
-; AVX1-NEXT: vpaddd %xmm5, %xmm6, %xmm5
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm6
-; AVX1-NEXT: vpshufb %xmm6, %xmm3, %xmm6
-; AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpshufb %xmm1, %xmm3, %xmm1
-; AVX1-NEXT: vpaddb %xmm6, %xmm1, %xmm1
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm1[2],xmm4[2],xmm1[3],xmm4[3]
-; AVX1-NEXT: vpsadbw %xmm4, %xmm6, %xmm6
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
-; AVX1-NEXT: vpackuswb %xmm6, %xmm1, %xmm1
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm6
-; AVX1-NEXT: vpshufb %xmm6, %xmm3, %xmm6
-; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; AVX1-NEXT: vpsadbw %xmm4, %xmm2, %xmm2
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
-; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpaddd %xmm0, %xmm5, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: vzeroupper
-; AVX1-NEXT: ret{{[l|q]}}
-;
-; AVX2-LABEL: reduce_ctpop_v16i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm3
-; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX2-NEXT: # ymm4 = mem[0,1,0,1]
-; AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
-; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
-; AVX2-NEXT: vpaddb %ymm3, %ymm1, %ymm1
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT: vpunpckhdq {{.*#+}} ymm5 = ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[6],ymm3[6],ymm1[7],ymm3[7]
-; AVX2-NEXT: vpsadbw %ymm3, %ymm5, %ymm5
-; AVX2-NEXT: vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5]
-; AVX2-NEXT: vpsadbw %ymm3, %ymm1, %ymm1
-; AVX2-NEXT: vpackuswb %ymm5, %ymm1, %ymm1
-; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm5
-; AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm5
-; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
-; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
-; AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm0
-; AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[6],ymm3[6],ymm0[7],ymm3[7]
-; AVX2-NEXT: vpsadbw %ymm3, %ymm2, %ymm2
-; AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]
-; AVX2-NEXT: vpsadbw %ymm3, %ymm0, %ymm0
-; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: ret{{[l|q]}}
-;
-; AVX512VL-LABEL: reduce_ctpop_v16i32:
-; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX1-LABEL: reduce_ctpop_v16i32:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm5
+; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; X86-AVX1-NEXT: vpaddb %xmm5, %xmm2, %xmm5
+; X86-AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero
+; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpackuswb %xmm6, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackuswb %xmm6, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero
+; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpackuswb %xmm6, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm5, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: vzeroupper
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: reduce_ctpop_v16i32:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
+; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm5
+; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; X64-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; X64-AVX1-NEXT: vpaddb %xmm5, %xmm2, %xmm5
+; X64-AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm6, %xmm6
+; X64-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm5, %xmm5
+; X64-AVX1-NEXT: vpackuswb %xmm6, %xmm5, %xmm5
+; X64-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm6
+; X64-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; X64-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm6, %xmm6
+; X64-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackuswb %xmm6, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm5, %xmm0, %xmm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm6
+; X64-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X64-AVX1-NEXT: vpsrlw $4, %xmm5, %xmm5
+; X64-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm5
+; X64-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; X64-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
+; X64-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm6 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm6, %xmm6
+; X64-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm5 = xmm5[0],zero,xmm5[1],zero
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm5, %xmm5
+; X64-AVX1-NEXT: vpackuswb %xmm6, %xmm5, %xmm5
+; X64-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm6
+; X64-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X64-AVX1-NEXT: vpaddb %xmm6, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm3 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm3, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm5, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX1-NEXT: vzeroupper
+; X64-AVX1-NEXT: retq
+;
+; X86-AVX2-LABEL: reduce_ctpop_v16i32:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX2-NEXT: vpand %ymm2, %ymm1, %ymm3
+; X86-AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX2-NEXT: # ymm4 = mem[0,1,0,1]
+; X86-AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3
+; X86-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
+; X86-AVX2-NEXT: vpaddb %ymm3, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X86-AVX2-NEXT: vpunpckhdq {{.*#+}} ymm5 = ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[6],ymm3[6],ymm1[7],ymm3[7]
+; X86-AVX2-NEXT: vpsadbw %ymm3, %ymm5, %ymm5
+; X86-AVX2-NEXT: vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5]
+; X86-AVX2-NEXT: vpsadbw %ymm3, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpackuswb %ymm5, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm5
+; X86-AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm5
+; X86-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
+; X86-AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[6],ymm3[6],ymm0[7],ymm3[7]
+; X86-AVX2-NEXT: vpsadbw %ymm3, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]
+; X86-AVX2-NEXT: vpsadbw %ymm3, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: vzeroupper
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: reduce_ctpop_v16i32:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX2-NEXT: vpand %ymm2, %ymm1, %ymm3
+; X64-AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX2-NEXT: # ymm4 = mem[0,1,0,1]
+; X64-AVX2-NEXT: vpshufb %ymm3, %ymm4, %ymm3
+; X64-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpand %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
+; X64-AVX2-NEXT: vpaddb %ymm3, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X64-AVX2-NEXT: vpunpckhdq {{.*#+}} ymm5 = ymm1[2],ymm3[2],ymm1[3],ymm3[3],ymm1[6],ymm3[6],ymm1[7],ymm3[7]
+; X64-AVX2-NEXT: vpsadbw %ymm3, %ymm5, %ymm5
+; X64-AVX2-NEXT: vpunpckldq {{.*#+}} ymm1 = ymm1[0],ymm3[0],ymm1[1],ymm3[1],ymm1[4],ymm3[4],ymm1[5],ymm3[5]
+; X64-AVX2-NEXT: vpsadbw %ymm3, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpackuswb %ymm5, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm5
+; X64-AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm5
+; X64-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpand %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
+; X64-AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[6],ymm3[6],ymm0[7],ymm3[7]
+; X64-AVX2-NEXT: vpsadbw %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[4],ymm3[4],ymm0[5],ymm3[5]
+; X64-AVX2-NEXT: vpsadbw %ymm3, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX2-NEXT: vzeroupper
+; X64-AVX2-NEXT: retq
+;
+; AVX512VL-LABEL: reduce_ctpop_v16i32:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpbroadcastb {{.*#+}} zmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
; AVX512VL-NEXT: vpandq %zmm1, %zmm0, %zmm2
; AVX512VL-NEXT: vbroadcasti32x4 {{.*#+}} zmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
; AVX512VL-NEXT: # zmm3 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
@@ -2118,7 +2300,8 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovd %xmm0, %eax
+; AVX512VL-NEXT: vmovq %xmm0, %rax
+; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
@@ -2130,7 +2313,8 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax
+; AVX512VPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VPOPCNT-NEXT: vzeroupper
; AVX512VPOPCNT-NEXT: retq
%p0 = tail call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %a0)
@@ -2145,134 +2329,144 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: movl %esp, %ebp
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm5
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm6
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm4
+; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm6
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm3
; X86-SSE2-NEXT: psrlw $1, %xmm3
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm3
-; X86-SSE2-NEXT: psubb %xmm3, %xmm6
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; X86-SSE2-NEXT: pand %xmm2, %xmm3
+; X86-SSE2-NEXT: psubb %xmm3, %xmm1
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm4
-; X86-SSE2-NEXT: pand %xmm3, %xmm4
-; X86-SSE2-NEXT: psrlw $2, %xmm6
-; X86-SSE2-NEXT: pand %xmm3, %xmm6
-; X86-SSE2-NEXT: paddb %xmm4, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm7
-; X86-SSE2-NEXT: psrlw $4, %xmm7
-; X86-SSE2-NEXT: paddb %xmm6, %xmm7
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
-; X86-SSE2-NEXT: psrlw $1, %xmm4
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm4
-; X86-SSE2-NEXT: psubb %xmm4, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
-; X86-SSE2-NEXT: pand %xmm3, %xmm4
-; X86-SSE2-NEXT: psrlw $2, %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: paddb %xmm4, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm6
-; X86-SSE2-NEXT: psrlw $4, %xmm6
-; X86-SSE2-NEXT: paddb %xmm1, %xmm6
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE2-NEXT: pand %xmm1, %xmm7
-; X86-SSE2-NEXT: pand %xmm1, %xmm6
-; X86-SSE2-NEXT: paddb %xmm7, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
-; X86-SSE2-NEXT: psrlw $1, %xmm1
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; X86-SSE2-NEXT: pand %xmm4, %xmm1
-; X86-SSE2-NEXT: psubb %xmm1, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: psrlw $2, %xmm5
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm5
; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: paddb %xmm1, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm7
-; X86-SSE2-NEXT: psrlw $4, %xmm7
-; X86-SSE2-NEXT: paddb %xmm5, %xmm7
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
-; X86-SSE2-NEXT: psrlw $1, %xmm1
-; X86-SSE2-NEXT: pand %xmm4, %xmm1
-; X86-SSE2-NEXT: psubb %xmm1, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
+; X86-SSE2-NEXT: psrlw $2, %xmm1
; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: psrlw $2, %xmm5
-; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: paddb %xmm1, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
-; X86-SSE2-NEXT: psrlw $4, %xmm1
; X86-SSE2-NEXT: paddb %xmm5, %xmm1
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE2-NEXT: pand %xmm4, %xmm7
-; X86-SSE2-NEXT: pand %xmm4, %xmm1
-; X86-SSE2-NEXT: paddb %xmm7, %xmm1
-; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm5
-; X86-SSE2-NEXT: paddb %xmm6, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm6
-; X86-SSE2-NEXT: psrlw $1, %xmm6
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; X86-SSE2-NEXT: pand %xmm4, %xmm6
-; X86-SSE2-NEXT: psubb %xmm6, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm6
-; X86-SSE2-NEXT: pand %xmm3, %xmm6
-; X86-SSE2-NEXT: psrlw $2, %xmm5
-; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: paddb %xmm6, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm7
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm7
; X86-SSE2-NEXT: psrlw $4, %xmm7
-; X86-SSE2-NEXT: paddb %xmm5, %xmm7
+; X86-SSE2-NEXT: paddb %xmm1, %xmm7
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm1, %xmm7
; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
; X86-SSE2-NEXT: psrlw $1, %xmm5
-; X86-SSE2-NEXT: pand %xmm4, %xmm5
+; X86-SSE2-NEXT: pand %xmm2, %xmm5
; X86-SSE2-NEXT: psubb %xmm5, %xmm0
; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
; X86-SSE2-NEXT: pand %xmm3, %xmm5
; X86-SSE2-NEXT: psrlw $2, %xmm0
; X86-SSE2-NEXT: pand %xmm3, %xmm0
; X86-SSE2-NEXT: paddb %xmm5, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm6
-; X86-SSE2-NEXT: psrlw $4, %xmm6
-; X86-SSE2-NEXT: paddb %xmm0, %xmm6
-; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
+; X86-SSE2-NEXT: psrlw $4, %xmm5
+; X86-SSE2-NEXT: paddb %xmm0, %xmm5
+; X86-SSE2-NEXT: pxor %xmm0, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm7
+; X86-SSE2-NEXT: pand %xmm1, %xmm5
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm5
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: packuswb %xmm7, %xmm5
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm7
+; X86-SSE2-NEXT: psrlw $1, %xmm7
+; X86-SSE2-NEXT: pand %xmm2, %xmm7
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: psubb %xmm7, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm7
+; X86-SSE2-NEXT: pand %xmm3, %xmm7
+; X86-SSE2-NEXT: psrlw $2, %xmm6
+; X86-SSE2-NEXT: pand %xmm3, %xmm6
+; X86-SSE2-NEXT: paddb %xmm7, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm7
+; X86-SSE2-NEXT: psrlw $4, %xmm7
+; X86-SSE2-NEXT: paddb %xmm6, %xmm7
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm6
+; X86-SSE2-NEXT: psrlw $1, %xmm6
+; X86-SSE2-NEXT: pand %xmm2, %xmm6
+; X86-SSE2-NEXT: psubb %xmm6, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm6
+; X86-SSE2-NEXT: pand %xmm3, %xmm6
+; X86-SSE2-NEXT: psrlw $2, %xmm4
+; X86-SSE2-NEXT: pand %xmm3, %xmm4
+; X86-SSE2-NEXT: paddb %xmm6, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm2
+; X86-SSE2-NEXT: psrlw $4, %xmm2
+; X86-SSE2-NEXT: paddb %xmm4, %xmm2
+; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm6
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
; X86-SSE2-NEXT: pand %xmm4, %xmm7
-; X86-SSE2-NEXT: pand %xmm4, %xmm6
-; X86-SSE2-NEXT: paddb %xmm7, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
-; X86-SSE2-NEXT: psrlw $1, %xmm5
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; X86-SSE2-NEXT: pand %xmm4, %xmm5
-; X86-SSE2-NEXT: psubb %xmm5, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
-; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: psrlw $2, %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: paddb %xmm5, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm7
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm7
+; X86-SSE2-NEXT: pand %xmm4, %xmm2
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm2
+; X86-SSE2-NEXT: packuswb %xmm7, %xmm2
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm4
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm5
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm2
+; X86-SSE2-NEXT: psrlw $1, %xmm2
+; X86-SSE2-NEXT: pand %xmm0, %xmm2
+; X86-SSE2-NEXT: psubb %xmm2, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm2
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: psrlw $2, %xmm4
+; X86-SSE2-NEXT: pand %xmm3, %xmm4
+; X86-SSE2-NEXT: paddb %xmm2, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm7
; X86-SSE2-NEXT: psrlw $4, %xmm7
-; X86-SSE2-NEXT: paddb %xmm0, %xmm7
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE2-NEXT: psrlw $1, %xmm0
-; X86-SSE2-NEXT: pand %xmm4, %xmm0
-; X86-SSE2-NEXT: psubb %xmm0, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: psrlw $2, %xmm2
+; X86-SSE2-NEXT: paddb %xmm4, %xmm7
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm2
+; X86-SSE2-NEXT: psrlw $1, %xmm2
+; X86-SSE2-NEXT: pand %xmm0, %xmm2
+; X86-SSE2-NEXT: psubb %xmm2, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm2
; X86-SSE2-NEXT: pand %xmm3, %xmm2
-; X86-SSE2-NEXT: paddb %xmm0, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE2-NEXT: psrlw $4, %xmm0
-; X86-SSE2-NEXT: paddb %xmm2, %xmm0
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE2-NEXT: pand %xmm2, %xmm7
-; X86-SSE2-NEXT: pand %xmm2, %xmm0
-; X86-SSE2-NEXT: paddb %xmm7, %xmm0
-; X86-SSE2-NEXT: paddb %xmm6, %xmm0
-; X86-SSE2-NEXT: paddb %xmm1, %xmm0
-; X86-SSE2-NEXT: pxor %xmm1, %xmm1
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm1, %xmm0
+; X86-SSE2-NEXT: psrlw $2, %xmm6
+; X86-SSE2-NEXT: pand %xmm3, %xmm6
+; X86-SSE2-NEXT: paddb %xmm2, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm2
+; X86-SSE2-NEXT: psrlw $4, %xmm2
+; X86-SSE2-NEXT: paddb %xmm6, %xmm2
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm6
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm4, %xmm7
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm7
+; X86-SSE2-NEXT: pand %xmm4, %xmm2
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm2
+; X86-SSE2-NEXT: packuswb %xmm7, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm4
+; X86-SSE2-NEXT: psrlw $1, %xmm4
+; X86-SSE2-NEXT: pand %xmm0, %xmm4
+; X86-SSE2-NEXT: psubb %xmm4, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm4
+; X86-SSE2-NEXT: pand %xmm3, %xmm4
+; X86-SSE2-NEXT: psrlw $2, %xmm6
+; X86-SSE2-NEXT: pand %xmm3, %xmm6
+; X86-SSE2-NEXT: paddb %xmm4, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm7
+; X86-SSE2-NEXT: psrlw $4, %xmm7
+; X86-SSE2-NEXT: paddb %xmm6, %xmm7
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm6
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm4
+; X86-SSE2-NEXT: psrlw $1, %xmm4
+; X86-SSE2-NEXT: pand %xmm0, %xmm4
+; X86-SSE2-NEXT: psubb %xmm4, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm4
+; X86-SSE2-NEXT: pand %xmm3, %xmm4
+; X86-SSE2-NEXT: psrlw $2, %xmm6
+; X86-SSE2-NEXT: pand %xmm3, %xmm6
+; X86-SSE2-NEXT: paddb %xmm4, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm3
+; X86-SSE2-NEXT: psrlw $4, %xmm3
+; X86-SSE2-NEXT: paddb %xmm6, %xmm3
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm0, %xmm7
+; X86-SSE2-NEXT: pand %xmm0, %xmm3
+; X86-SSE2-NEXT: pxor %xmm0, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm7
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm7, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm3, %xmm2
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm5
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm5
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; X86-SSE2-NEXT: paddq %xmm5, %xmm0
; X86-SSE2-NEXT: movd %xmm0, %eax
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; X86-SSE2-NEXT: movd %xmm0, %edx
@@ -2282,125 +2476,133 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
;
; X64-SSE2-LABEL: reduce_ctpop_v16i64:
; X64-SSE2: # %bb.0:
-; X64-SSE2-NEXT: movdqa %xmm5, %xmm8
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm8
; X64-SSE2-NEXT: psrlw $1, %xmm8
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm9 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; X64-SSE2-NEXT: pand %xmm9, %xmm8
-; X64-SSE2-NEXT: psubb %xmm8, %xmm5
+; X64-SSE2-NEXT: psubb %xmm8, %xmm1
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm8 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; X64-SSE2-NEXT: movdqa %xmm5, %xmm10
-; X64-SSE2-NEXT: pand %xmm8, %xmm10
-; X64-SSE2-NEXT: psrlw $2, %xmm5
-; X64-SSE2-NEXT: pand %xmm8, %xmm5
-; X64-SSE2-NEXT: paddb %xmm10, %xmm5
-; X64-SSE2-NEXT: movdqa %xmm5, %xmm11
-; X64-SSE2-NEXT: psrlw $4, %xmm11
-; X64-SSE2-NEXT: paddb %xmm5, %xmm11
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-SSE2-NEXT: pand %xmm5, %xmm11
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm10
-; X64-SSE2-NEXT: psrlw $1, %xmm10
-; X64-SSE2-NEXT: pand %xmm9, %xmm10
-; X64-SSE2-NEXT: psubb %xmm10, %xmm1
; X64-SSE2-NEXT: movdqa %xmm1, %xmm10
; X64-SSE2-NEXT: pand %xmm8, %xmm10
; X64-SSE2-NEXT: psrlw $2, %xmm1
; X64-SSE2-NEXT: pand %xmm8, %xmm1
; X64-SSE2-NEXT: paddb %xmm10, %xmm1
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm10
-; X64-SSE2-NEXT: psrlw $4, %xmm10
-; X64-SSE2-NEXT: paddb %xmm1, %xmm10
-; X64-SSE2-NEXT: pand %xmm5, %xmm10
-; X64-SSE2-NEXT: paddb %xmm11, %xmm10
-; X64-SSE2-NEXT: movdqa %xmm7, %xmm1
-; X64-SSE2-NEXT: psrlw $1, %xmm1
-; X64-SSE2-NEXT: pand %xmm9, %xmm1
-; X64-SSE2-NEXT: psubb %xmm1, %xmm7
-; X64-SSE2-NEXT: movdqa %xmm7, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
-; X64-SSE2-NEXT: psrlw $2, %xmm7
-; X64-SSE2-NEXT: pand %xmm8, %xmm7
-; X64-SSE2-NEXT: paddb %xmm1, %xmm7
-; X64-SSE2-NEXT: movdqa %xmm7, %xmm11
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm12
+; X64-SSE2-NEXT: psrlw $4, %xmm12
+; X64-SSE2-NEXT: paddb %xmm1, %xmm12
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm10 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-SSE2-NEXT: pand %xmm10, %xmm12
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm12
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm11
+; X64-SSE2-NEXT: psrlw $1, %xmm11
+; X64-SSE2-NEXT: pand %xmm9, %xmm11
+; X64-SSE2-NEXT: psubb %xmm11, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm11
+; X64-SSE2-NEXT: pand %xmm8, %xmm11
+; X64-SSE2-NEXT: psrlw $2, %xmm0
+; X64-SSE2-NEXT: pand %xmm8, %xmm0
+; X64-SSE2-NEXT: paddb %xmm11, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm11
; X64-SSE2-NEXT: psrlw $4, %xmm11
-; X64-SSE2-NEXT: paddb %xmm7, %xmm11
-; X64-SSE2-NEXT: pand %xmm5, %xmm11
-; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
-; X64-SSE2-NEXT: psrlw $1, %xmm1
-; X64-SSE2-NEXT: pand %xmm9, %xmm1
-; X64-SSE2-NEXT: psubb %xmm1, %xmm3
-; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
+; X64-SSE2-NEXT: paddb %xmm0, %xmm11
+; X64-SSE2-NEXT: pand %xmm10, %xmm11
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm11
+; X64-SSE2-NEXT: packuswb %xmm12, %xmm11
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm0
+; X64-SSE2-NEXT: psrlw $1, %xmm0
+; X64-SSE2-NEXT: pand %xmm9, %xmm0
+; X64-SSE2-NEXT: psubb %xmm0, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm0
+; X64-SSE2-NEXT: pand %xmm8, %xmm0
; X64-SSE2-NEXT: psrlw $2, %xmm3
; X64-SSE2-NEXT: pand %xmm8, %xmm3
-; X64-SSE2-NEXT: paddb %xmm1, %xmm3
-; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
-; X64-SSE2-NEXT: psrlw $4, %xmm1
-; X64-SSE2-NEXT: paddb %xmm3, %xmm1
-; X64-SSE2-NEXT: pand %xmm5, %xmm1
-; X64-SSE2-NEXT: paddb %xmm11, %xmm1
-; X64-SSE2-NEXT: paddb %xmm10, %xmm1
-; X64-SSE2-NEXT: movdqa %xmm4, %xmm3
+; X64-SSE2-NEXT: paddb %xmm0, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm0
+; X64-SSE2-NEXT: psrlw $4, %xmm0
+; X64-SSE2-NEXT: paddb %xmm3, %xmm0
+; X64-SSE2-NEXT: pand %xmm10, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
; X64-SSE2-NEXT: psrlw $1, %xmm3
; X64-SSE2-NEXT: pand %xmm9, %xmm3
-; X64-SSE2-NEXT: psubb %xmm3, %xmm4
-; X64-SSE2-NEXT: movdqa %xmm4, %xmm3
+; X64-SSE2-NEXT: psubb %xmm3, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
; X64-SSE2-NEXT: pand %xmm8, %xmm3
+; X64-SSE2-NEXT: psrlw $2, %xmm2
+; X64-SSE2-NEXT: pand %xmm8, %xmm2
+; X64-SSE2-NEXT: paddb %xmm3, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
+; X64-SSE2-NEXT: psrlw $4, %xmm3
+; X64-SSE2-NEXT: paddb %xmm2, %xmm3
+; X64-SSE2-NEXT: pand %xmm10, %xmm3
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm0, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm3, %xmm11
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm0
+; X64-SSE2-NEXT: psrlw $1, %xmm0
+; X64-SSE2-NEXT: pand %xmm9, %xmm0
+; X64-SSE2-NEXT: psubb %xmm0, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm0
+; X64-SSE2-NEXT: pand %xmm8, %xmm0
+; X64-SSE2-NEXT: psrlw $2, %xmm5
+; X64-SSE2-NEXT: pand %xmm8, %xmm5
+; X64-SSE2-NEXT: paddb %xmm0, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm2
+; X64-SSE2-NEXT: psrlw $4, %xmm2
+; X64-SSE2-NEXT: paddb %xmm5, %xmm2
+; X64-SSE2-NEXT: pand %xmm10, %xmm2
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm4, %xmm0
+; X64-SSE2-NEXT: psrlw $1, %xmm0
+; X64-SSE2-NEXT: pand %xmm9, %xmm0
+; X64-SSE2-NEXT: psubb %xmm0, %xmm4
+; X64-SSE2-NEXT: movdqa %xmm4, %xmm0
+; X64-SSE2-NEXT: pand %xmm8, %xmm0
; X64-SSE2-NEXT: psrlw $2, %xmm4
; X64-SSE2-NEXT: pand %xmm8, %xmm4
-; X64-SSE2-NEXT: paddb %xmm3, %xmm4
-; X64-SSE2-NEXT: movdqa %xmm4, %xmm7
-; X64-SSE2-NEXT: psrlw $4, %xmm7
-; X64-SSE2-NEXT: paddb %xmm4, %xmm7
-; X64-SSE2-NEXT: pand %xmm5, %xmm7
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE2-NEXT: paddb %xmm0, %xmm4
+; X64-SSE2-NEXT: movdqa %xmm4, %xmm0
+; X64-SSE2-NEXT: psrlw $4, %xmm0
+; X64-SSE2-NEXT: paddb %xmm4, %xmm0
+; X64-SSE2-NEXT: pand %xmm10, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm2, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm2
+; X64-SSE2-NEXT: psrlw $1, %xmm2
+; X64-SSE2-NEXT: pand %xmm9, %xmm2
+; X64-SSE2-NEXT: psubb %xmm2, %xmm7
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm2
+; X64-SSE2-NEXT: pand %xmm8, %xmm2
+; X64-SSE2-NEXT: psrlw $2, %xmm7
+; X64-SSE2-NEXT: pand %xmm8, %xmm7
+; X64-SSE2-NEXT: paddb %xmm2, %xmm7
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm2
+; X64-SSE2-NEXT: psrlw $4, %xmm2
+; X64-SSE2-NEXT: paddb %xmm7, %xmm2
+; X64-SSE2-NEXT: pand %xmm10, %xmm2
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm6, %xmm3
; X64-SSE2-NEXT: psrlw $1, %xmm3
; X64-SSE2-NEXT: pand %xmm9, %xmm3
-; X64-SSE2-NEXT: psubb %xmm3, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE2-NEXT: psubb %xmm3, %xmm6
+; X64-SSE2-NEXT: movdqa %xmm6, %xmm3
; X64-SSE2-NEXT: pand %xmm8, %xmm3
-; X64-SSE2-NEXT: psrlw $2, %xmm0
-; X64-SSE2-NEXT: pand %xmm8, %xmm0
-; X64-SSE2-NEXT: paddb %xmm3, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE2-NEXT: psrlw $4, %xmm3
-; X64-SSE2-NEXT: paddb %xmm0, %xmm3
-; X64-SSE2-NEXT: pand %xmm5, %xmm3
-; X64-SSE2-NEXT: paddb %xmm7, %xmm3
-; X64-SSE2-NEXT: movdqa %xmm6, %xmm0
-; X64-SSE2-NEXT: psrlw $1, %xmm0
-; X64-SSE2-NEXT: pand %xmm9, %xmm0
-; X64-SSE2-NEXT: psubb %xmm0, %xmm6
-; X64-SSE2-NEXT: movdqa %xmm6, %xmm0
-; X64-SSE2-NEXT: pand %xmm8, %xmm0
; X64-SSE2-NEXT: psrlw $2, %xmm6
; X64-SSE2-NEXT: pand %xmm8, %xmm6
-; X64-SSE2-NEXT: paddb %xmm0, %xmm6
-; X64-SSE2-NEXT: movdqa %xmm6, %xmm0
-; X64-SSE2-NEXT: psrlw $4, %xmm0
-; X64-SSE2-NEXT: paddb %xmm6, %xmm0
-; X64-SSE2-NEXT: pand %xmm5, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm2, %xmm4
-; X64-SSE2-NEXT: psrlw $1, %xmm4
-; X64-SSE2-NEXT: pand %xmm9, %xmm4
-; X64-SSE2-NEXT: psubb %xmm4, %xmm2
-; X64-SSE2-NEXT: movdqa %xmm2, %xmm4
-; X64-SSE2-NEXT: pand %xmm8, %xmm4
-; X64-SSE2-NEXT: psrlw $2, %xmm2
-; X64-SSE2-NEXT: pand %xmm8, %xmm2
-; X64-SSE2-NEXT: paddb %xmm4, %xmm2
-; X64-SSE2-NEXT: movdqa %xmm2, %xmm4
-; X64-SSE2-NEXT: psrlw $4, %xmm4
-; X64-SSE2-NEXT: paddb %xmm2, %xmm4
-; X64-SSE2-NEXT: pand %xmm5, %xmm4
-; X64-SSE2-NEXT: paddb %xmm0, %xmm4
-; X64-SSE2-NEXT: paddb %xmm3, %xmm4
-; X64-SSE2-NEXT: paddb %xmm1, %xmm4
-; X64-SSE2-NEXT: pxor %xmm0, %xmm0
-; X64-SSE2-NEXT: psadbw %xmm4, %xmm0
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm0, %xmm1
-; X64-SSE2-NEXT: movq %xmm1, %rax
+; X64-SSE2-NEXT: paddb %xmm3, %xmm6
+; X64-SSE2-NEXT: movdqa %xmm6, %xmm3
+; X64-SSE2-NEXT: psrlw $4, %xmm3
+; X64-SSE2-NEXT: paddb %xmm6, %xmm3
+; X64-SSE2-NEXT: pand %xmm10, %xmm3
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm2, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm3, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm0, %xmm11
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm11
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm11[2,3,2,3]
+; X64-SSE2-NEXT: paddq %xmm11, %xmm0
+; X64-SSE2-NEXT: movq %xmm0, %rax
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: reduce_ctpop_v16i64:
@@ -2408,101 +2610,110 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE4-NEXT: pushl %ebp
; X86-SSE4-NEXT: movl %esp, %ebp
; X86-SSE4-NEXT: andl $-16, %esp
-; X86-SSE4-NEXT: subl $32, %esp
-; X86-SSE4-NEXT: movaps %xmm2, (%esp) # 16-byte Spill
-; X86-SSE4-NEXT: movdqa %xmm0, %xmm2
-; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm5
-; X86-SSE4-NEXT: movdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE4-NEXT: movdqa %xmm5, %xmm6
-; X86-SSE4-NEXT: pand %xmm4, %xmm6
-; X86-SSE4-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm6, %xmm7
-; X86-SSE4-NEXT: psrlw $4, %xmm5
-; X86-SSE4-NEXT: pand %xmm4, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE4-NEXT: pshufb %xmm5, %xmm6
-; X86-SSE4-NEXT: paddb %xmm7, %xmm6
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm5
-; X86-SSE4-NEXT: pand %xmm4, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm5, %xmm7
-; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm5
-; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm1
-; X86-SSE4-NEXT: paddb %xmm7, %xmm5
-; X86-SSE4-NEXT: paddb %xmm6, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm6
-; X86-SSE4-NEXT: pand %xmm4, %xmm6
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm6, %xmm7
-; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm6
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm1
-; X86-SSE4-NEXT: paddb %xmm7, %xmm6
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm7
-; X86-SSE4-NEXT: pand %xmm4, %xmm7
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
-; X86-SSE4-NEXT: pshufb %xmm7, %xmm0
-; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm7
-; X86-SSE4-NEXT: paddb %xmm0, %xmm7
-; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm0
-; X86-SSE4-NEXT: paddb %xmm6, %xmm7
-; X86-SSE4-NEXT: paddb %xmm5, %xmm7
-; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm5
-; X86-SSE4-NEXT: psrlw $4, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm6
-; X86-SSE4-NEXT: paddb %xmm5, %xmm6
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
+; X86-SSE4-NEXT: subl $16, %esp
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm5
+; X86-SSE4-NEXT: movdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm6
+; X86-SSE4-NEXT: pand %xmm4, %xmm6
+; X86-SSE4-NEXT: movdqa {{.*#+}} xmm0 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm6, %xmm7
+; X86-SSE4-NEXT: psrlw $4, %xmm1
+; X86-SSE4-NEXT: pand %xmm4, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm6
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm6
+; X86-SSE4-NEXT: paddb %xmm7, %xmm6
; X86-SSE4-NEXT: movdqa %xmm3, %xmm1
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm1
+; X86-SSE4-NEXT: pand %xmm4, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm7
+; X86-SSE4-NEXT: psrlw $4, %xmm3
+; X86-SSE4-NEXT: pand %xmm4, %xmm3
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE4-NEXT: pshufb %xmm3, %xmm1
+; X86-SSE4-NEXT: pxor %xmm3, %xmm3
+; X86-SSE4-NEXT: psadbw %xmm3, %xmm6
+; X86-SSE4-NEXT: paddb %xmm7, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm3, %xmm1
+; X86-SSE4-NEXT: packusdw %xmm6, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm5, %xmm6
+; X86-SSE4-NEXT: pand %xmm4, %xmm6
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm6, %xmm7
+; X86-SSE4-NEXT: psrlw $4, %xmm5
+; X86-SSE4-NEXT: pand %xmm4, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm6
+; X86-SSE4-NEXT: pshufb %xmm5, %xmm6
+; X86-SSE4-NEXT: paddb %xmm7, %xmm6
+; X86-SSE4-NEXT: movdqa %xmm2, %xmm5
+; X86-SSE4-NEXT: pand %xmm4, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm5, %xmm7
; X86-SSE4-NEXT: psrlw $4, %xmm2
; X86-SSE4-NEXT: pand %xmm4, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm5
-; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm2
-; X86-SSE4-NEXT: paddb %xmm1, %xmm5
-; X86-SSE4-NEXT: paddb %xmm6, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm1
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE4-NEXT: pshufb %xmm2, %xmm3
+; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm5
+; X86-SSE4-NEXT: paddb %xmm7, %xmm3
+; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm2
+; X86-SSE4-NEXT: pxor %xmm7, %xmm7
+; X86-SSE4-NEXT: psadbw %xmm7, %xmm6
+; X86-SSE4-NEXT: psadbw %xmm7, %xmm3
+; X86-SSE4-NEXT: packusdw %xmm6, %xmm3
+; X86-SSE4-NEXT: packusdw %xmm3, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm2, %xmm3
+; X86-SSE4-NEXT: pand %xmm4, %xmm3
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm3, %xmm7
; X86-SSE4-NEXT: psrlw $4, %xmm2
; X86-SSE4-NEXT: pand %xmm4, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm0
-; X86-SSE4-NEXT: paddb %xmm1, %xmm0
-; X86-SSE4-NEXT: movdqa (%esp), %xmm6 # 16-byte Reload
-; X86-SSE4-NEXT: movdqa %xmm6, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm2
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm2
-; X86-SSE4-NEXT: psrlw $4, %xmm6
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm6
+; X86-SSE4-NEXT: pshufb %xmm2, %xmm6
+; X86-SSE4-NEXT: paddb %xmm7, %xmm6
+; X86-SSE4-NEXT: movdqa %xmm5, %xmm2
+; X86-SSE4-NEXT: pand %xmm4, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE4-NEXT: pshufb %xmm2, %xmm3
+; X86-SSE4-NEXT: psrlw $4, %xmm5
+; X86-SSE4-NEXT: pand %xmm4, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE4-NEXT: pshufb %xmm5, %xmm2
+; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm7
+; X86-SSE4-NEXT: pxor %xmm5, %xmm5
+; X86-SSE4-NEXT: psadbw %xmm5, %xmm6
+; X86-SSE4-NEXT: paddb %xmm3, %xmm2
+; X86-SSE4-NEXT: psadbw %xmm5, %xmm2
+; X86-SSE4-NEXT: packusdw %xmm6, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm7, %xmm3
+; X86-SSE4-NEXT: pand %xmm4, %xmm3
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm6
+; X86-SSE4-NEXT: pshufb %xmm3, %xmm6
+; X86-SSE4-NEXT: psrlw $4, %xmm7
+; X86-SSE4-NEXT: pand %xmm4, %xmm7
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm5
+; X86-SSE4-NEXT: pshufb %xmm7, %xmm5
+; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm3
+; X86-SSE4-NEXT: paddb %xmm6, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
; X86-SSE4-NEXT: pand %xmm4, %xmm6
-; X86-SSE4-NEXT: pshufb %xmm6, %xmm3
-; X86-SSE4-NEXT: paddb %xmm2, %xmm3
-; X86-SSE4-NEXT: paddb %xmm0, %xmm3
-; X86-SSE4-NEXT: paddb %xmm5, %xmm3
-; X86-SSE4-NEXT: paddb %xmm7, %xmm3
-; X86-SSE4-NEXT: pxor %xmm0, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm6, %xmm7
+; X86-SSE4-NEXT: psrlw $4, %xmm3
+; X86-SSE4-NEXT: pand %xmm4, %xmm3
+; X86-SSE4-NEXT: pshufb %xmm3, %xmm0
+; X86-SSE4-NEXT: paddb %xmm7, %xmm0
+; X86-SSE4-NEXT: pxor %xmm3, %xmm3
+; X86-SSE4-NEXT: psadbw %xmm3, %xmm5
; X86-SSE4-NEXT: psadbw %xmm3, %xmm0
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm0, %xmm1
-; X86-SSE4-NEXT: movd %xmm1, %eax
-; X86-SSE4-NEXT: pextrd $1, %xmm1, %edx
+; X86-SSE4-NEXT: packusdw %xmm5, %xmm0
+; X86-SSE4-NEXT: packusdw %xmm0, %xmm2
+; X86-SSE4-NEXT: packuswb %xmm2, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm3, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: pextrd $1, %xmm0, %edx
; X86-SSE4-NEXT: movl %ebp, %esp
; X86-SSE4-NEXT: popl %ebp
; X86-SSE4-NEXT: retl
@@ -2510,90 +2721,98 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X64-SSE4-LABEL: reduce_ctpop_v16i64:
; X64-SSE4: # %bb.0:
; X64-SSE4-NEXT: movdqa {{.*#+}} xmm9 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-SSE4-NEXT: movdqa %xmm5, %xmm10
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm10
; X64-SSE4-NEXT: pand %xmm9, %xmm10
; X64-SSE4-NEXT: movdqa {{.*#+}} xmm8 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
; X64-SSE4-NEXT: pshufb %xmm10, %xmm11
-; X64-SSE4-NEXT: psrlw $4, %xmm5
-; X64-SSE4-NEXT: pand %xmm9, %xmm5
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm10
-; X64-SSE4-NEXT: pshufb %xmm5, %xmm10
-; X64-SSE4-NEXT: paddb %xmm11, %xmm10
-; X64-SSE4-NEXT: movdqa %xmm1, %xmm5
-; X64-SSE4-NEXT: pand %xmm9, %xmm5
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
-; X64-SSE4-NEXT: pshufb %xmm5, %xmm11
; X64-SSE4-NEXT: psrlw $4, %xmm1
; X64-SSE4-NEXT: pand %xmm9, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm5
-; X64-SSE4-NEXT: pshufb %xmm1, %xmm5
-; X64-SSE4-NEXT: paddb %xmm11, %xmm5
-; X64-SSE4-NEXT: paddb %xmm10, %xmm5
-; X64-SSE4-NEXT: movdqa %xmm7, %xmm1
-; X64-SSE4-NEXT: pand %xmm9, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm12
+; X64-SSE4-NEXT: pshufb %xmm1, %xmm12
+; X64-SSE4-NEXT: paddb %xmm11, %xmm12
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm12
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm10
+; X64-SSE4-NEXT: pand %xmm9, %xmm10
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
+; X64-SSE4-NEXT: pshufb %xmm10, %xmm11
+; X64-SSE4-NEXT: psrlw $4, %xmm0
+; X64-SSE4-NEXT: pand %xmm9, %xmm0
; X64-SSE4-NEXT: movdqa %xmm8, %xmm10
-; X64-SSE4-NEXT: pshufb %xmm1, %xmm10
-; X64-SSE4-NEXT: psrlw $4, %xmm7
-; X64-SSE4-NEXT: pand %xmm9, %xmm7
+; X64-SSE4-NEXT: pshufb %xmm0, %xmm10
+; X64-SSE4-NEXT: paddb %xmm11, %xmm10
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm10
+; X64-SSE4-NEXT: packusdw %xmm12, %xmm10
+; X64-SSE4-NEXT: movdqa %xmm3, %xmm0
+; X64-SSE4-NEXT: pand %xmm9, %xmm0
; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
-; X64-SSE4-NEXT: pshufb %xmm7, %xmm11
-; X64-SSE4-NEXT: paddb %xmm10, %xmm11
-; X64-SSE4-NEXT: movdqa %xmm3, %xmm1
-; X64-SSE4-NEXT: pand %xmm9, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm7
-; X64-SSE4-NEXT: pshufb %xmm1, %xmm7
+; X64-SSE4-NEXT: pshufb %xmm0, %xmm11
; X64-SSE4-NEXT: psrlw $4, %xmm3
; X64-SSE4-NEXT: pand %xmm9, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm1
-; X64-SSE4-NEXT: pshufb %xmm3, %xmm1
-; X64-SSE4-NEXT: paddb %xmm7, %xmm1
-; X64-SSE4-NEXT: paddb %xmm11, %xmm1
-; X64-SSE4-NEXT: paddb %xmm5, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm4, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm0
+; X64-SSE4-NEXT: pshufb %xmm3, %xmm0
+; X64-SSE4-NEXT: paddb %xmm11, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm2, %xmm3
; X64-SSE4-NEXT: pand %xmm9, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm5
-; X64-SSE4-NEXT: pshufb %xmm3, %xmm5
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
+; X64-SSE4-NEXT: pshufb %xmm3, %xmm11
+; X64-SSE4-NEXT: psrlw $4, %xmm2
+; X64-SSE4-NEXT: pand %xmm9, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm3
+; X64-SSE4-NEXT: pshufb %xmm2, %xmm3
+; X64-SSE4-NEXT: paddb %xmm11, %xmm3
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm3
+; X64-SSE4-NEXT: packusdw %xmm0, %xmm3
+; X64-SSE4-NEXT: packusdw %xmm3, %xmm10
+; X64-SSE4-NEXT: movdqa %xmm5, %xmm0
+; X64-SSE4-NEXT: pand %xmm9, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm2
+; X64-SSE4-NEXT: pshufb %xmm0, %xmm2
+; X64-SSE4-NEXT: psrlw $4, %xmm5
+; X64-SSE4-NEXT: pand %xmm9, %xmm5
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm3
+; X64-SSE4-NEXT: pshufb %xmm5, %xmm3
+; X64-SSE4-NEXT: paddb %xmm2, %xmm3
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm4, %xmm0
+; X64-SSE4-NEXT: pand %xmm9, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm2
+; X64-SSE4-NEXT: pshufb %xmm0, %xmm2
; X64-SSE4-NEXT: psrlw $4, %xmm4
; X64-SSE4-NEXT: pand %xmm9, %xmm4
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm7
-; X64-SSE4-NEXT: pshufb %xmm4, %xmm7
-; X64-SSE4-NEXT: paddb %xmm5, %xmm7
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm0
+; X64-SSE4-NEXT: pshufb %xmm4, %xmm0
+; X64-SSE4-NEXT: paddb %xmm2, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm3, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm7, %xmm2
+; X64-SSE4-NEXT: pand %xmm9, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm3
+; X64-SSE4-NEXT: pshufb %xmm2, %xmm3
+; X64-SSE4-NEXT: psrlw $4, %xmm7
+; X64-SSE4-NEXT: pand %xmm9, %xmm7
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm2
+; X64-SSE4-NEXT: pshufb %xmm7, %xmm2
+; X64-SSE4-NEXT: paddb %xmm3, %xmm2
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm6, %xmm3
; X64-SSE4-NEXT: pand %xmm9, %xmm3
; X64-SSE4-NEXT: movdqa %xmm8, %xmm4
; X64-SSE4-NEXT: pshufb %xmm3, %xmm4
-; X64-SSE4-NEXT: psrlw $4, %xmm0
-; X64-SSE4-NEXT: pand %xmm9, %xmm0
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm3
-; X64-SSE4-NEXT: pshufb %xmm0, %xmm3
-; X64-SSE4-NEXT: paddb %xmm4, %xmm3
-; X64-SSE4-NEXT: paddb %xmm7, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm6, %xmm0
-; X64-SSE4-NEXT: pand %xmm9, %xmm0
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm4
-; X64-SSE4-NEXT: pshufb %xmm0, %xmm4
; X64-SSE4-NEXT: psrlw $4, %xmm6
; X64-SSE4-NEXT: pand %xmm9, %xmm6
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm0
-; X64-SSE4-NEXT: pshufb %xmm6, %xmm0
-; X64-SSE4-NEXT: paddb %xmm4, %xmm0
-; X64-SSE4-NEXT: movdqa %xmm2, %xmm4
-; X64-SSE4-NEXT: pand %xmm9, %xmm4
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm5
-; X64-SSE4-NEXT: pshufb %xmm4, %xmm5
-; X64-SSE4-NEXT: psrlw $4, %xmm2
-; X64-SSE4-NEXT: pand %xmm9, %xmm2
-; X64-SSE4-NEXT: pshufb %xmm2, %xmm8
-; X64-SSE4-NEXT: paddb %xmm5, %xmm8
-; X64-SSE4-NEXT: paddb %xmm0, %xmm8
-; X64-SSE4-NEXT: paddb %xmm3, %xmm8
-; X64-SSE4-NEXT: paddb %xmm1, %xmm8
-; X64-SSE4-NEXT: pxor %xmm0, %xmm0
-; X64-SSE4-NEXT: psadbw %xmm8, %xmm0
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm0, %xmm1
-; X64-SSE4-NEXT: movq %xmm1, %rax
+; X64-SSE4-NEXT: pshufb %xmm6, %xmm8
+; X64-SSE4-NEXT: paddb %xmm4, %xmm8
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm8
+; X64-SSE4-NEXT: packusdw %xmm2, %xmm8
+; X64-SSE4-NEXT: packusdw %xmm8, %xmm0
+; X64-SSE4-NEXT: packuswb %xmm0, %xmm10
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm10
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm10[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm10, %xmm0
+; X64-SSE4-NEXT: movq %xmm0, %rax
; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: reduce_ctpop_v16i64:
@@ -2601,9 +2820,8 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X86-AVX1-NEXT: pushl %ebp
; X86-AVX1-NEXT: movl %esp, %ebp
; X86-AVX1-NEXT: andl $-32, %esp
-; X86-AVX1-NEXT: subl $96, %esp
-; X86-AVX1-NEXT: vmovaps %ymm1, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
-; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X86-AVX1-NEXT: subl $32, %esp
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5
; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm6
; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
@@ -2611,16 +2829,51 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X86-AVX1-NEXT: vpsrlw $4, %xmm5, %xmm5
; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm5
; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm6
+; X86-AVX1-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpsadbw %xmm5, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
+; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm6, %xmm0, %xmm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm6
; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm7
; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
; X86-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm6
; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
; X86-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm6, %xmm5
-; X86-AVX1-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-AVX1-NEXT: vpsadbw %xmm5, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
+; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm5, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm6, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm1
+; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm5, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpsadbw %xmm5, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
+; X86-AVX1-NEXT: vmovdqa 8(%ebp), %xmm2
; X86-AVX1-NEXT: vmovdqa 24(%ebp), %xmm6
; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm7
; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
@@ -2628,47 +2881,18 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm6
; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
; X86-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm7
-; X86-AVX1-NEXT: vpand %xmm3, %xmm7, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X86-AVX1-NEXT: vpsrlw $4, %xmm7, %xmm7
-; X86-AVX1-NEXT: vpand %xmm3, %xmm7, %xmm7
+; X86-AVX1-NEXT: vpsadbw %xmm5, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm7
; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm7, %xmm5
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpaddb {{[-0-9]+}}(%e{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm2, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm2
-; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vmovdqa 8(%ebp), %xmm2
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm2, %xmm1
-; X86-AVX1-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %ymm6 # 32-byte Reload
-; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm2
-; X86-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
-; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm3
-; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
-; X86-AVX1-NEXT: vpaddb %xmm2, %xmm3, %xmm2
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm2, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpsadbw %xmm5, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpackusdw %xmm6, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm5, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
@@ -2680,7 +2904,7 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
;
; X64-AVX1-LABEL: reduce_ctpop_v16i64:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
; X64-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm7
; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
@@ -2688,61 +2912,69 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X64-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
; X64-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm6
; X64-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7
-; X64-AVX1-NEXT: vpand %xmm4, %xmm7, %xmm8
+; X64-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm7
+; X64-AVX1-NEXT: vpxor %xmm6, %xmm6, %xmm6
+; X64-AVX1-NEXT: vpsadbw %xmm6, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm8
; X64-AVX1-NEXT: vpshufb %xmm8, %xmm5, %xmm8
-; X64-AVX1-NEXT: vpsrlw $4, %xmm7, %xmm7
-; X64-AVX1-NEXT: vpand %xmm4, %xmm7, %xmm7
-; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm8, %xmm7
-; X64-AVX1-NEXT: vpaddb %xmm6, %xmm7, %xmm6
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm7
+; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
+; X64-AVX1-NEXT: vpaddb %xmm0, %xmm8, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm6, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm7, %xmm0, %xmm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm7
; X64-AVX1-NEXT: vpand %xmm4, %xmm7, %xmm8
; X64-AVX1-NEXT: vpshufb %xmm8, %xmm5, %xmm8
; X64-AVX1-NEXT: vpsrlw $4, %xmm7, %xmm7
; X64-AVX1-NEXT: vpand %xmm4, %xmm7, %xmm7
; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
; X64-AVX1-NEXT: vpaddb %xmm7, %xmm8, %xmm7
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm8
-; X64-AVX1-NEXT: vpand %xmm4, %xmm8, %xmm9
-; X64-AVX1-NEXT: vpshufb %xmm9, %xmm5, %xmm9
-; X64-AVX1-NEXT: vpsrlw $4, %xmm8, %xmm8
-; X64-AVX1-NEXT: vpand %xmm4, %xmm8, %xmm8
+; X64-AVX1-NEXT: vpsadbw %xmm6, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm8
; X64-AVX1-NEXT: vpshufb %xmm8, %xmm5, %xmm8
-; X64-AVX1-NEXT: vpaddb %xmm9, %xmm8, %xmm8
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm8, %xmm7
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
+; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
+; X64-AVX1-NEXT: vpaddb %xmm1, %xmm8, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm6, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm7, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm1
+; X64-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm7
+; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
+; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
+; X64-AVX1-NEXT: vpaddb %xmm7, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm6, %xmm1, %xmm1
; X64-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm7
; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
; X64-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
; X64-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2
; X64-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
; X64-AVX1-NEXT: vpaddb %xmm7, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm7
+; X64-AVX1-NEXT: vpsadbw %xmm6, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpackusdw %xmm1, %xmm2, %xmm1
+; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2
+; X64-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm7
; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
-; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpand %xmm4, %xmm3, %xmm2
+; X64-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2
; X64-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
+; X64-AVX1-NEXT: vpaddb %xmm7, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpsadbw %xmm6, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpand %xmm4, %xmm3, %xmm7
+; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
; X64-AVX1-NEXT: vpsrlw $4, %xmm3, %xmm3
; X64-AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3
; X64-AVX1-NEXT: vpshufb %xmm3, %xmm5, %xmm3
-; X64-AVX1-NEXT: vpaddb %xmm2, %xmm3, %xmm2
-; X64-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm3
-; X64-AVX1-NEXT: vpshufb %xmm3, %xmm5, %xmm3
-; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm2, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpaddb %xmm7, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpsadbw %xmm6, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpackusdw %xmm2, %xmm3, %xmm2
+; X64-AVX1-NEXT: vpackusdw %xmm2, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm6, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
@@ -2757,39 +2989,47 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X86-AVX2-NEXT: subl $32, %esp
; X86-AVX2-NEXT: vmovdqa 8(%ebp), %ymm5
; X86-AVX2-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm6
+; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm6
; X86-AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
; X86-AVX2-NEXT: # ymm4 = mem[0,1,0,1]
; X86-AVX2-NEXT: vpshufb %ymm6, %ymm4, %ymm6
-; X86-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
-; X86-AVX2-NEXT: vpaddb %ymm6, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpand %ymm3, %ymm0, %ymm6
-; X86-AVX2-NEXT: vpshufb %ymm6, %ymm4, %ymm6
+; X86-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
+; X86-AVX2-NEXT: vpaddb %ymm6, %ymm1, %ymm6
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm6, %ymm6
+; X86-AVX2-NEXT: vpand %ymm3, %ymm0, %ymm7
+; X86-AVX2-NEXT: vpshufb %ymm7, %ymm4, %ymm7
; X86-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
; X86-AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0
; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
-; X86-AVX2-NEXT: vpaddb %ymm6, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpand %ymm3, %ymm5, %ymm2
-; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
+; X86-AVX2-NEXT: vpaddb %ymm7, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackusdw %ymm6, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpand %ymm3, %ymm5, %ymm6
+; X86-AVX2-NEXT: vpshufb %ymm6, %ymm4, %ymm6
; X86-AVX2-NEXT: vpsrlw $4, %ymm5, %ymm5
; X86-AVX2-NEXT: vpand %ymm3, %ymm5, %ymm5
; X86-AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm5
-; X86-AVX2-NEXT: vpaddb %ymm2, %ymm5, %ymm2
-; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm5
-; X86-AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm5
-; X86-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
-; X86-AVX2-NEXT: vpaddb %ymm5, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpaddb %ymm2, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpaddb %ymm6, %ymm5, %ymm5
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm5, %ymm5
+; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm6
+; X86-AVX2-NEXT: vpshufb %ymm6, %ymm4, %ymm6
+; X86-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
+; X86-AVX2-NEXT: vpaddb %ymm6, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm1
+; X86-AVX2-NEXT: vpackusdw %ymm5, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
@@ -2802,39 +3042,47 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X64-AVX2-LABEL: reduce_ctpop_v16i64:
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpbroadcastb {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-AVX2-NEXT: vpand %ymm4, %ymm2, %ymm5
+; X64-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm5
; X64-AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
; X64-AVX2-NEXT: # ymm6 = mem[0,1,0,1]
; X64-AVX2-NEXT: vpshufb %ymm5, %ymm6, %ymm5
-; X64-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
-; X64-AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2
-; X64-AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm2
-; X64-AVX2-NEXT: vpaddb %ymm5, %ymm2, %ymm2
-; X64-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm5
-; X64-AVX2-NEXT: vpshufb %ymm5, %ymm6, %ymm5
+; X64-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpshufb %ymm1, %ymm6, %ymm1
+; X64-AVX2-NEXT: vpaddb %ymm5, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; X64-AVX2-NEXT: vpsadbw %ymm5, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm7
+; X64-AVX2-NEXT: vpshufb %ymm7, %ymm6, %ymm7
; X64-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
; X64-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
; X64-AVX2-NEXT: vpshufb %ymm0, %ymm6, %ymm0
-; X64-AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm2
-; X64-AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm2
+; X64-AVX2-NEXT: vpaddb %ymm7, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpsadbw %ymm5, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm1
+; X64-AVX2-NEXT: vpshufb %ymm1, %ymm6, %ymm1
; X64-AVX2-NEXT: vpsrlw $4, %ymm3, %ymm3
; X64-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
; X64-AVX2-NEXT: vpshufb %ymm3, %ymm6, %ymm3
-; X64-AVX2-NEXT: vpaddb %ymm2, %ymm3, %ymm2
-; X64-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm3
+; X64-AVX2-NEXT: vpaddb %ymm1, %ymm3, %ymm1
+; X64-AVX2-NEXT: vpsadbw %ymm5, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpand %ymm4, %ymm2, %ymm3
; X64-AVX2-NEXT: vpshufb %ymm3, %ymm6, %ymm3
-; X64-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
-; X64-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
-; X64-AVX2-NEXT: vpshufb %ymm1, %ymm6, %ymm1
-; X64-AVX2-NEXT: vpaddb %ymm3, %ymm1, %ymm1
-; X64-AVX2-NEXT: vpaddb %ymm2, %ymm1, %ymm1
-; X64-AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm2
+; X64-AVX2-NEXT: vpaddb %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpsadbw %ymm5, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm2, %ymm1
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovq %xmm0, %rax
@@ -2902,139 +3150,146 @@ define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64>
; X86-SSE2-NEXT: movl %esp, %ebp
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
; X86-SSE2-NEXT: movdqa %xmm0, %xmm4
-; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm6
; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm7
; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
; X86-SSE2-NEXT: psrlw $1, %xmm0
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; X86-SSE2-NEXT: pand %xmm5, %xmm0
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm6 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; X86-SSE2-NEXT: pand %xmm6, %xmm0
; X86-SSE2-NEXT: psubb %xmm0, %xmm1
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: pand %xmm5, %xmm0
; X86-SSE2-NEXT: psrlw $2, %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: paddb %xmm1, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
-; X86-SSE2-NEXT: psrlw $4, %xmm1
+; X86-SSE2-NEXT: pand %xmm5, %xmm1
; X86-SSE2-NEXT: paddb %xmm0, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm4, %xmm0
-; X86-SSE2-NEXT: psrlw $1, %xmm0
-; X86-SSE2-NEXT: pand %xmm5, %xmm0
-; X86-SSE2-NEXT: psubb %xmm0, %xmm4
-; X86-SSE2-NEXT: movdqa %xmm4, %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: psrlw $2, %xmm4
-; X86-SSE2-NEXT: pand %xmm3, %xmm4
-; X86-SSE2-NEXT: paddb %xmm0, %xmm4
-; X86-SSE2-NEXT: movdqa %xmm4, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
; X86-SSE2-NEXT: psrlw $4, %xmm0
-; X86-SSE2-NEXT: paddb %xmm0, %xmm4
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE2-NEXT: pand %xmm0, %xmm1
-; X86-SSE2-NEXT: pand %xmm0, %xmm4
+; X86-SSE2-NEXT: paddb %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm2, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm1
+; X86-SSE2-NEXT: psrlw $1, %xmm1
+; X86-SSE2-NEXT: pand %xmm6, %xmm1
+; X86-SSE2-NEXT: psubb %xmm1, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm1
+; X86-SSE2-NEXT: pand %xmm5, %xmm1
+; X86-SSE2-NEXT: psrlw $2, %xmm4
+; X86-SSE2-NEXT: pand %xmm5, %xmm4
; X86-SSE2-NEXT: paddb %xmm1, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm1
+; X86-SSE2-NEXT: psrlw $4, %xmm1
+; X86-SSE2-NEXT: paddb %xmm1, %xmm4
+; X86-SSE2-NEXT: pxor %xmm6, %xmm6
+; X86-SSE2-NEXT: psadbw %xmm6, %xmm0
+; X86-SSE2-NEXT: pand %xmm2, %xmm4
+; X86-SSE2-NEXT: psadbw %xmm6, %xmm4
+; X86-SSE2-NEXT: packuswb %xmm0, %xmm4
; X86-SSE2-NEXT: movdqa %xmm7, %xmm0
; X86-SSE2-NEXT: psrlw $1, %xmm0
-; X86-SSE2-NEXT: pand %xmm5, %xmm0
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; X86-SSE2-NEXT: pand %xmm2, %xmm0
; X86-SSE2-NEXT: psubb %xmm0, %xmm7
; X86-SSE2-NEXT: movdqa %xmm7, %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: pand %xmm5, %xmm0
; X86-SSE2-NEXT: psrlw $2, %xmm7
-; X86-SSE2-NEXT: pand %xmm3, %xmm7
+; X86-SSE2-NEXT: pand %xmm5, %xmm7
; X86-SSE2-NEXT: paddb %xmm0, %xmm7
-; X86-SSE2-NEXT: movdqa %xmm7, %xmm1
-; X86-SSE2-NEXT: psrlw $4, %xmm1
-; X86-SSE2-NEXT: paddb %xmm7, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE2-NEXT: psrlw $1, %xmm0
-; X86-SSE2-NEXT: pand %xmm5, %xmm0
-; X86-SSE2-NEXT: psubb %xmm0, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: psrlw $2, %xmm2
-; X86-SSE2-NEXT: pand %xmm3, %xmm2
-; X86-SSE2-NEXT: paddb %xmm0, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm7
+; X86-SSE2-NEXT: movdqa %xmm7, %xmm0
+; X86-SSE2-NEXT: psrlw $4, %xmm0
+; X86-SSE2-NEXT: paddb %xmm7, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm1
+; X86-SSE2-NEXT: psrlw $1, %xmm1
+; X86-SSE2-NEXT: pand %xmm2, %xmm1
+; X86-SSE2-NEXT: psubb %xmm1, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm1
+; X86-SSE2-NEXT: pand %xmm5, %xmm1
+; X86-SSE2-NEXT: psrlw $2, %xmm3
+; X86-SSE2-NEXT: pand %xmm5, %xmm3
+; X86-SSE2-NEXT: paddb %xmm1, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm7
; X86-SSE2-NEXT: psrlw $4, %xmm7
-; X86-SSE2-NEXT: paddb %xmm2, %xmm7
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm2
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE2-NEXT: pand %xmm0, %xmm1
-; X86-SSE2-NEXT: pand %xmm0, %xmm7
-; X86-SSE2-NEXT: paddb %xmm1, %xmm7
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: paddb %xmm3, %xmm7
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm1
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm2, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm6, %xmm0
+; X86-SSE2-NEXT: pand %xmm2, %xmm7
+; X86-SSE2-NEXT: psadbw %xmm6, %xmm7
+; X86-SSE2-NEXT: packuswb %xmm0, %xmm7
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
; X86-SSE2-NEXT: psrlw $1, %xmm0
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; X86-SSE2-NEXT: pand %xmm2, %xmm0
+; X86-SSE2-NEXT: psubb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
; X86-SSE2-NEXT: pand %xmm5, %xmm0
-; X86-SSE2-NEXT: psubb %xmm0, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: psrlw $2, %xmm2
-; X86-SSE2-NEXT: pand %xmm3, %xmm2
-; X86-SSE2-NEXT: paddb %xmm0, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm5, %xmm1
+; X86-SSE2-NEXT: paddb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
; X86-SSE2-NEXT: psrlw $4, %xmm0
-; X86-SSE2-NEXT: paddb %xmm2, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm1
+; X86-SSE2-NEXT: paddb %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm3
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm1
; X86-SSE2-NEXT: psrlw $1, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm2
+; X86-SSE2-NEXT: pand %xmm2, %xmm1
+; X86-SSE2-NEXT: psubb %xmm1, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm1
; X86-SSE2-NEXT: pand %xmm5, %xmm1
-; X86-SSE2-NEXT: psubb %xmm1, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: psrlw $2, %xmm6
-; X86-SSE2-NEXT: pand %xmm3, %xmm6
-; X86-SSE2-NEXT: paddb %xmm1, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm1
+; X86-SSE2-NEXT: psrlw $2, %xmm3
+; X86-SSE2-NEXT: pand %xmm5, %xmm3
+; X86-SSE2-NEXT: paddb %xmm1, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm1
; X86-SSE2-NEXT: psrlw $4, %xmm1
-; X86-SSE2-NEXT: paddb %xmm6, %xmm1
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm6
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE2-NEXT: pand %xmm5, %xmm0
-; X86-SSE2-NEXT: pand %xmm5, %xmm1
-; X86-SSE2-NEXT: paddb %xmm0, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm0
-; X86-SSE2-NEXT: psrlw $1, %xmm0
+; X86-SSE2-NEXT: paddb %xmm3, %xmm1
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
; X86-SSE2-NEXT: pand %xmm2, %xmm0
-; X86-SSE2-NEXT: psubb %xmm0, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: psrlw $2, %xmm6
-; X86-SSE2-NEXT: pand %xmm3, %xmm6
-; X86-SSE2-NEXT: paddb %xmm0, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm2
+; X86-SSE2-NEXT: psadbw %xmm6, %xmm0
+; X86-SSE2-NEXT: pand %xmm2, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm6, %xmm1
+; X86-SSE2-NEXT: packuswb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE2-NEXT: psrlw $1, %xmm2
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm2
+; X86-SSE2-NEXT: psubb %xmm2, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE2-NEXT: pand %xmm5, %xmm2
+; X86-SSE2-NEXT: psrlw $2, %xmm0
+; X86-SSE2-NEXT: pand %xmm5, %xmm0
+; X86-SSE2-NEXT: paddb %xmm2, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm2
; X86-SSE2-NEXT: psrlw $4, %xmm2
-; X86-SSE2-NEXT: paddb %xmm6, %xmm2
+; X86-SSE2-NEXT: paddb %xmm0, %xmm2
; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm6
-; X86-SSE2-NEXT: psrlw $1, %xmm6
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm6
-; X86-SSE2-NEXT: psubb %xmm6, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm6
-; X86-SSE2-NEXT: pand %xmm3, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE2-NEXT: psrlw $1, %xmm3
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm3
+; X86-SSE2-NEXT: psubb %xmm3, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE2-NEXT: pand %xmm5, %xmm3
; X86-SSE2-NEXT: psrlw $2, %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: paddb %xmm6, %xmm0
+; X86-SSE2-NEXT: pand %xmm5, %xmm0
+; X86-SSE2-NEXT: paddb %xmm3, %xmm0
; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
; X86-SSE2-NEXT: psrlw $4, %xmm3
; X86-SSE2-NEXT: paddb %xmm0, %xmm3
-; X86-SSE2-NEXT: pand %xmm5, %xmm2
-; X86-SSE2-NEXT: pand %xmm5, %xmm3
-; X86-SSE2-NEXT: paddb %xmm2, %xmm3
-; X86-SSE2-NEXT: pxor %xmm0, %xmm0
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm4
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm7
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm3
-; X86-SSE2-NEXT: movdqa %xmm4, %xmm0
-; X86-SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm7[1]
-; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm7[0]
-; X86-SSE2-NEXT: paddq %xmm0, %xmm4
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
-; X86-SSE2-NEXT: punpckhqdq {{.*#+}} xmm0 = xmm0[1],xmm3[1]
-; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm3[0]
-; X86-SSE2-NEXT: paddq %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm0, %xmm2
+; X86-SSE2-NEXT: pand %xmm0, %xmm3
+; X86-SSE2-NEXT: psadbw %xmm6, %xmm2
+; X86-SSE2-NEXT: psadbw %xmm6, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm6, %xmm4
+; X86-SSE2-NEXT: packuswb %xmm6, %xmm7
+; X86-SSE2-NEXT: packuswb %xmm7, %xmm4
+; X86-SSE2-NEXT: packuswb %xmm6, %xmm1
+; X86-SSE2-NEXT: packuswb %xmm6, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm3, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm6, %xmm4
+; X86-SSE2-NEXT: psadbw %xmm6, %xmm1
; X86-SSE2-NEXT: movdqa %xmm4, %xmm0
; X86-SSE2-NEXT: movl %ebp, %esp
; X86-SSE2-NEXT: popl %ebp
@@ -3044,128 +3299,132 @@ define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64>
; X64-SSE2: # %bb.0:
; X64-SSE2-NEXT: movdqa %xmm1, %xmm8
; X64-SSE2-NEXT: psrlw $1, %xmm8
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm9 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; X64-SSE2-NEXT: pand %xmm9, %xmm8
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm10 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; X64-SSE2-NEXT: pand %xmm10, %xmm8
; X64-SSE2-NEXT: psubb %xmm8, %xmm1
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm8 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm10
-; X64-SSE2-NEXT: pand %xmm8, %xmm10
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm9 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm8
+; X64-SSE2-NEXT: pand %xmm9, %xmm8
; X64-SSE2-NEXT: psrlw $2, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
-; X64-SSE2-NEXT: paddb %xmm1, %xmm10
-; X64-SSE2-NEXT: movdqa %xmm10, %xmm1
-; X64-SSE2-NEXT: psrlw $4, %xmm1
-; X64-SSE2-NEXT: paddb %xmm10, %xmm1
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm10 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-SSE2-NEXT: pand %xmm9, %xmm1
+; X64-SSE2-NEXT: paddb %xmm8, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm12
+; X64-SSE2-NEXT: psrlw $4, %xmm12
+; X64-SSE2-NEXT: paddb %xmm1, %xmm12
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm11 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-SSE2-NEXT: pand %xmm11, %xmm12
+; X64-SSE2-NEXT: pxor %xmm8, %xmm8
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm12
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT: psrlw $1, %xmm1
; X64-SSE2-NEXT: pand %xmm10, %xmm1
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm11
-; X64-SSE2-NEXT: psrlw $1, %xmm11
-; X64-SSE2-NEXT: pand %xmm9, %xmm11
-; X64-SSE2-NEXT: psubb %xmm11, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm11
-; X64-SSE2-NEXT: pand %xmm8, %xmm11
+; X64-SSE2-NEXT: psubb %xmm1, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT: pand %xmm9, %xmm1
; X64-SSE2-NEXT: psrlw $2, %xmm0
-; X64-SSE2-NEXT: pand %xmm8, %xmm0
-; X64-SSE2-NEXT: paddb %xmm11, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm11
-; X64-SSE2-NEXT: psrlw $4, %xmm11
-; X64-SSE2-NEXT: paddb %xmm11, %xmm0
-; X64-SSE2-NEXT: pand %xmm10, %xmm0
+; X64-SSE2-NEXT: pand %xmm9, %xmm0
+; X64-SSE2-NEXT: paddb %xmm1, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT: psrlw $4, %xmm1
; X64-SSE2-NEXT: paddb %xmm1, %xmm0
+; X64-SSE2-NEXT: pand %xmm11, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm12, %xmm0
; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
; X64-SSE2-NEXT: psrlw $1, %xmm1
-; X64-SSE2-NEXT: pand %xmm9, %xmm1
+; X64-SSE2-NEXT: pand %xmm10, %xmm1
; X64-SSE2-NEXT: psubb %xmm1, %xmm3
; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
+; X64-SSE2-NEXT: pand %xmm9, %xmm1
; X64-SSE2-NEXT: psrlw $2, %xmm3
-; X64-SSE2-NEXT: pand %xmm8, %xmm3
+; X64-SSE2-NEXT: pand %xmm9, %xmm3
; X64-SSE2-NEXT: paddb %xmm1, %xmm3
; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
; X64-SSE2-NEXT: psrlw $4, %xmm1
; X64-SSE2-NEXT: paddb %xmm3, %xmm1
-; X64-SSE2-NEXT: pand %xmm10, %xmm1
+; X64-SSE2-NEXT: pand %xmm11, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm1
; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
; X64-SSE2-NEXT: psrlw $1, %xmm3
-; X64-SSE2-NEXT: pand %xmm9, %xmm3
+; X64-SSE2-NEXT: pand %xmm10, %xmm3
; X64-SSE2-NEXT: psubb %xmm3, %xmm2
; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
-; X64-SSE2-NEXT: pand %xmm8, %xmm3
+; X64-SSE2-NEXT: pand %xmm9, %xmm3
; X64-SSE2-NEXT: psrlw $2, %xmm2
-; X64-SSE2-NEXT: pand %xmm8, %xmm2
+; X64-SSE2-NEXT: pand %xmm9, %xmm2
; X64-SSE2-NEXT: paddb %xmm3, %xmm2
; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
; X64-SSE2-NEXT: psrlw $4, %xmm3
; X64-SSE2-NEXT: paddb %xmm2, %xmm3
-; X64-SSE2-NEXT: pand %xmm10, %xmm3
-; X64-SSE2-NEXT: paddb %xmm1, %xmm3
+; X64-SSE2-NEXT: pand %xmm11, %xmm3
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm3
; X64-SSE2-NEXT: movdqa %xmm5, %xmm1
; X64-SSE2-NEXT: psrlw $1, %xmm1
-; X64-SSE2-NEXT: pand %xmm9, %xmm1
+; X64-SSE2-NEXT: pand %xmm10, %xmm1
; X64-SSE2-NEXT: psubb %xmm1, %xmm5
; X64-SSE2-NEXT: movdqa %xmm5, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
+; X64-SSE2-NEXT: pand %xmm9, %xmm1
; X64-SSE2-NEXT: psrlw $2, %xmm5
-; X64-SSE2-NEXT: pand %xmm8, %xmm5
-; X64-SSE2-NEXT: paddb %xmm5, %xmm1
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X64-SSE2-NEXT: pand %xmm9, %xmm5
+; X64-SSE2-NEXT: paddb %xmm1, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm2
; X64-SSE2-NEXT: psrlw $4, %xmm2
-; X64-SSE2-NEXT: paddb %xmm1, %xmm2
-; X64-SSE2-NEXT: pand %xmm10, %xmm2
+; X64-SSE2-NEXT: paddb %xmm5, %xmm2
+; X64-SSE2-NEXT: pand %xmm11, %xmm2
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm2
; X64-SSE2-NEXT: movdqa %xmm4, %xmm1
; X64-SSE2-NEXT: psrlw $1, %xmm1
-; X64-SSE2-NEXT: pand %xmm9, %xmm1
+; X64-SSE2-NEXT: pand %xmm10, %xmm1
; X64-SSE2-NEXT: psubb %xmm1, %xmm4
; X64-SSE2-NEXT: movdqa %xmm4, %xmm5
-; X64-SSE2-NEXT: pand %xmm8, %xmm5
+; X64-SSE2-NEXT: pand %xmm9, %xmm5
; X64-SSE2-NEXT: psrlw $2, %xmm4
-; X64-SSE2-NEXT: pand %xmm8, %xmm4
+; X64-SSE2-NEXT: pand %xmm9, %xmm4
; X64-SSE2-NEXT: paddb %xmm4, %xmm5
; X64-SSE2-NEXT: movdqa %xmm5, %xmm1
; X64-SSE2-NEXT: psrlw $4, %xmm1
; X64-SSE2-NEXT: paddb %xmm5, %xmm1
-; X64-SSE2-NEXT: pand %xmm10, %xmm1
-; X64-SSE2-NEXT: paddb %xmm2, %xmm1
+; X64-SSE2-NEXT: pand %xmm11, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm2, %xmm1
; X64-SSE2-NEXT: movdqa %xmm7, %xmm2
; X64-SSE2-NEXT: psrlw $1, %xmm2
-; X64-SSE2-NEXT: pand %xmm9, %xmm2
+; X64-SSE2-NEXT: pand %xmm10, %xmm2
; X64-SSE2-NEXT: psubb %xmm2, %xmm7
; X64-SSE2-NEXT: movdqa %xmm7, %xmm2
-; X64-SSE2-NEXT: pand %xmm8, %xmm2
+; X64-SSE2-NEXT: pand %xmm9, %xmm2
; X64-SSE2-NEXT: psrlw $2, %xmm7
-; X64-SSE2-NEXT: pand %xmm8, %xmm7
+; X64-SSE2-NEXT: pand %xmm9, %xmm7
; X64-SSE2-NEXT: paddb %xmm2, %xmm7
; X64-SSE2-NEXT: movdqa %xmm7, %xmm2
; X64-SSE2-NEXT: psrlw $4, %xmm2
; X64-SSE2-NEXT: paddb %xmm7, %xmm2
-; X64-SSE2-NEXT: pand %xmm10, %xmm2
+; X64-SSE2-NEXT: pand %xmm11, %xmm2
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm2
; X64-SSE2-NEXT: movdqa %xmm6, %xmm4
; X64-SSE2-NEXT: psrlw $1, %xmm4
-; X64-SSE2-NEXT: pand %xmm9, %xmm4
+; X64-SSE2-NEXT: pand %xmm10, %xmm4
; X64-SSE2-NEXT: psubb %xmm4, %xmm6
; X64-SSE2-NEXT: movdqa %xmm6, %xmm4
-; X64-SSE2-NEXT: pand %xmm8, %xmm4
+; X64-SSE2-NEXT: pand %xmm9, %xmm4
; X64-SSE2-NEXT: psrlw $2, %xmm6
-; X64-SSE2-NEXT: pand %xmm8, %xmm6
+; X64-SSE2-NEXT: pand %xmm9, %xmm6
; X64-SSE2-NEXT: paddb %xmm4, %xmm6
; X64-SSE2-NEXT: movdqa %xmm6, %xmm4
; X64-SSE2-NEXT: psrlw $4, %xmm4
; X64-SSE2-NEXT: paddb %xmm6, %xmm4
-; X64-SSE2-NEXT: pand %xmm10, %xmm4
-; X64-SSE2-NEXT: paddb %xmm2, %xmm4
-; X64-SSE2-NEXT: pxor %xmm2, %xmm2
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm0
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm3
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm1
-; X64-SSE2-NEXT: psadbw %xmm2, %xmm4
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE2-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1]
-; X64-SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; X64-SSE2-NEXT: paddq %xmm2, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm2
-; X64-SSE2-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm4[1]
-; X64-SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
-; X64-SSE2-NEXT: paddq %xmm2, %xmm1
+; X64-SSE2-NEXT: pand %xmm11, %xmm4
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm4
+; X64-SSE2-NEXT: packuswb %xmm2, %xmm4
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm3, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm4
+; X64-SSE2-NEXT: packuswb %xmm4, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm1
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: reduce_ctpop_v4i64_buildvector_v4i64:
@@ -3173,109 +3432,116 @@ define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64>
; X86-SSE4-NEXT: pushl %ebp
; X86-SSE4-NEXT: movl %esp, %ebp
; X86-SSE4-NEXT: andl $-16, %esp
-; X86-SSE4-NEXT: subl $16, %esp
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm5
+; X86-SSE4-NEXT: subl $32, %esp
+; X86-SSE4-NEXT: movaps %xmm2, (%esp) # 16-byte Spill
; X86-SSE4-NEXT: movdqa %xmm0, %xmm3
-; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm6
-; X86-SSE4-NEXT: movdqa {{.*#+}} xmm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm6
+; X86-SSE4-NEXT: movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
-; X86-SSE4-NEXT: pand %xmm7, %xmm0
+; X86-SSE4-NEXT: pand %xmm5, %xmm0
; X86-SSE4-NEXT: movdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
; X86-SSE4-NEXT: movdqa %xmm4, %xmm2
; X86-SSE4-NEXT: pshufb %xmm0, %xmm2
; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm7, %xmm1
+; X86-SSE4-NEXT: pand %xmm5, %xmm1
; X86-SSE4-NEXT: movdqa %xmm4, %xmm7
; X86-SSE4-NEXT: pshufb %xmm1, %xmm7
; X86-SSE4-NEXT: paddb %xmm2, %xmm7
; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
-; X86-SSE4-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE4-NEXT: pand %xmm2, %xmm0
+; X86-SSE4-NEXT: pand %xmm5, %xmm0
; X86-SSE4-NEXT: movdqa %xmm4, %xmm1
; X86-SSE4-NEXT: pshufb %xmm0, %xmm1
; X86-SSE4-NEXT: psrlw $4, %xmm3
-; X86-SSE4-NEXT: pand %xmm2, %xmm3
-; X86-SSE4-NEXT: movdqa %xmm4, %xmm0
-; X86-SSE4-NEXT: pshufb %xmm3, %xmm0
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm2
-; X86-SSE4-NEXT: paddb %xmm1, %xmm0
-; X86-SSE4-NEXT: paddb %xmm7, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm1
-; X86-SSE4-NEXT: movdqa {{.*#+}} xmm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE4-NEXT: pand %xmm7, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm4, %xmm3
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm3
-; X86-SSE4-NEXT: psrlw $4, %xmm2
-; X86-SSE4-NEXT: pand %xmm7, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm4, %xmm1
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm1
-; X86-SSE4-NEXT: paddb %xmm3, %xmm1
+; X86-SSE4-NEXT: pand %xmm5, %xmm3
; X86-SSE4-NEXT: movdqa %xmm5, %xmm2
-; X86-SSE4-NEXT: pand %xmm7, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm7, %xmm3
-; X86-SSE4-NEXT: movdqa %xmm4, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm7
-; X86-SSE4-NEXT: psrlw $4, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm2
-; X86-SSE4-NEXT: pand %xmm3, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm4, %xmm3
-; X86-SSE4-NEXT: pshufb %xmm5, %xmm3
-; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm5
-; X86-SSE4-NEXT: paddb %xmm7, %xmm3
-; X86-SSE4-NEXT: paddb %xmm1, %xmm3
-; X86-SSE4-NEXT: movdqa %xmm5, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm5
+; X86-SSE4-NEXT: pshufb %xmm3, %xmm5
+; X86-SSE4-NEXT: pxor %xmm0, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm7
+; X86-SSE4-NEXT: paddb %xmm1, %xmm5
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm5
+; X86-SSE4-NEXT: packusdw %xmm7, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm2, %xmm3
; X86-SSE4-NEXT: pand %xmm2, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm4, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm7
-; X86-SSE4-NEXT: psrlw $4, %xmm5
-; X86-SSE4-NEXT: pand %xmm2, %xmm5
; X86-SSE4-NEXT: movdqa %xmm4, %xmm2
-; X86-SSE4-NEXT: pshufb %xmm5, %xmm2
-; X86-SSE4-NEXT: paddb %xmm7, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm6, %xmm1
-; X86-SSE4-NEXT: movdqa {{.*#+}} xmm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE4-NEXT: pand %xmm7, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm4, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm5
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm2
; X86-SSE4-NEXT: psrlw $4, %xmm6
-; X86-SSE4-NEXT: pand %xmm7, %xmm6
+; X86-SSE4-NEXT: pand %xmm3, %xmm6
; X86-SSE4-NEXT: movdqa %xmm4, %xmm1
; X86-SSE4-NEXT: pshufb %xmm6, %xmm1
-; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm6
-; X86-SSE4-NEXT: paddb %xmm5, %xmm1
; X86-SSE4-NEXT: paddb %xmm2, %xmm1
+; X86-SSE4-NEXT: movdqa (%esp), %xmm6 # 16-byte Reload
; X86-SSE4-NEXT: movdqa %xmm6, %xmm2
-; X86-SSE4-NEXT: pand %xmm7, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm4, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm5
-; X86-SSE4-NEXT: psrlw $4, %xmm6
-; X86-SSE4-NEXT: pand %xmm7, %xmm6
-; X86-SSE4-NEXT: movdqa %xmm4, %xmm2
-; X86-SSE4-NEXT: pshufb %xmm6, %xmm2
-; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm6
-; X86-SSE4-NEXT: paddb %xmm5, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm6, %xmm5
-; X86-SSE4-NEXT: pand %xmm7, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm4, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm5, %xmm7
+; X86-SSE4-NEXT: pand %xmm3, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm0
+; X86-SSE4-NEXT: pshufb %xmm2, %xmm0
; X86-SSE4-NEXT: psrlw $4, %xmm6
-; X86-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm6
-; X86-SSE4-NEXT: pshufb %xmm6, %xmm4
-; X86-SSE4-NEXT: paddb %xmm7, %xmm4
-; X86-SSE4-NEXT: paddb %xmm2, %xmm4
+; X86-SSE4-NEXT: pand %xmm3, %xmm6
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm6
+; X86-SSE4-NEXT: pshufb %xmm2, %xmm6
+; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm7
; X86-SSE4-NEXT: pxor %xmm2, %xmm2
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm0
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm3
; X86-SSE4-NEXT: psadbw %xmm2, %xmm1
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm4
+; X86-SSE4-NEXT: paddb %xmm0, %xmm6
+; X86-SSE4-NEXT: psadbw %xmm2, %xmm6
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm6
+; X86-SSE4-NEXT: movdqa %xmm7, %xmm0
+; X86-SSE4-NEXT: pand %xmm3, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm1
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm1
+; X86-SSE4-NEXT: psrlw $4, %xmm7
+; X86-SSE4-NEXT: pand %xmm3, %xmm7
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm2
+; X86-SSE4-NEXT: pshufb %xmm7, %xmm2
+; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm0
+; X86-SSE4-NEXT: paddb %xmm1, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE4-NEXT: pand %xmm3, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm7
+; X86-SSE4-NEXT: psrlw $4, %xmm0
+; X86-SSE4-NEXT: pand %xmm3, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm1
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm1
+; X86-SSE4-NEXT: paddb %xmm7, %xmm1
+; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm0
+; X86-SSE4-NEXT: pxor %xmm7, %xmm7
+; X86-SSE4-NEXT: psadbw %xmm7, %xmm2
+; X86-SSE4-NEXT: psadbw %xmm7, %xmm1
+; X86-SSE4-NEXT: packusdw %xmm2, %xmm1
; X86-SSE4-NEXT: movdqa %xmm0, %xmm2
-; X86-SSE4-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1]
-; X86-SSE4-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; X86-SSE4-NEXT: paddq %xmm2, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm2
-; X86-SSE4-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm4[1]
-; X86-SSE4-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
-; X86-SSE4-NEXT: paddq %xmm2, %xmm1
+; X86-SSE4-NEXT: pand %xmm3, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm2, %xmm7
+; X86-SSE4-NEXT: psrlw $4, %xmm0
+; X86-SSE4-NEXT: pand %xmm3, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm2
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm2
+; X86-SSE4-NEXT: paddb %xmm7, %xmm2
+; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm0
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm7
+; X86-SSE4-NEXT: pand %xmm3, %xmm7
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm3
+; X86-SSE4-NEXT: pshufb %xmm7, %xmm3
+; X86-SSE4-NEXT: psrlw $4, %xmm0
+; X86-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm4
+; X86-SSE4-NEXT: paddb %xmm3, %xmm4
+; X86-SSE4-NEXT: pxor %xmm0, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm2
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm4
+; X86-SSE4-NEXT: packusdw %xmm2, %xmm4
+; X86-SSE4-NEXT: packusdw %xmm0, %xmm5
+; X86-SSE4-NEXT: packusdw %xmm0, %xmm6
+; X86-SSE4-NEXT: packuswb %xmm6, %xmm5
+; X86-SSE4-NEXT: packusdw %xmm0, %xmm1
+; X86-SSE4-NEXT: packusdw %xmm0, %xmm4
+; X86-SSE4-NEXT: packuswb %xmm4, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm5
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm5, %xmm0
; X86-SSE4-NEXT: movl %ebp, %esp
; X86-SSE4-NEXT: popl %ebp
; X86-SSE4-NEXT: retl
@@ -3283,96 +3549,100 @@ define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64>
; X64-SSE4-LABEL: reduce_ctpop_v4i64_buildvector_v4i64:
; X64-SSE4: # %bb.0:
; X64-SSE4-NEXT: movdqa %xmm0, %xmm8
-; X64-SSE4-NEXT: movdqa {{.*#+}} xmm10 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-SSE4-NEXT: movdqa {{.*#+}} xmm11 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
; X64-SSE4-NEXT: movdqa %xmm1, %xmm0
-; X64-SSE4-NEXT: pand %xmm10, %xmm0
+; X64-SSE4-NEXT: pand %xmm11, %xmm0
; X64-SSE4-NEXT: movdqa {{.*#+}} xmm9 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X64-SSE4-NEXT: movdqa %xmm9, %xmm11
-; X64-SSE4-NEXT: pshufb %xmm0, %xmm11
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm10
+; X64-SSE4-NEXT: pshufb %xmm0, %xmm10
; X64-SSE4-NEXT: psrlw $4, %xmm1
-; X64-SSE4-NEXT: pand %xmm10, %xmm1
+; X64-SSE4-NEXT: pand %xmm11, %xmm1
; X64-SSE4-NEXT: movdqa %xmm9, %xmm12
; X64-SSE4-NEXT: pshufb %xmm1, %xmm12
-; X64-SSE4-NEXT: paddb %xmm11, %xmm12
+; X64-SSE4-NEXT: paddb %xmm10, %xmm12
+; X64-SSE4-NEXT: pxor %xmm10, %xmm10
+; X64-SSE4-NEXT: psadbw %xmm10, %xmm12
; X64-SSE4-NEXT: movdqa %xmm8, %xmm0
-; X64-SSE4-NEXT: pand %xmm10, %xmm0
+; X64-SSE4-NEXT: pand %xmm11, %xmm0
; X64-SSE4-NEXT: movdqa %xmm9, %xmm1
; X64-SSE4-NEXT: pshufb %xmm0, %xmm1
; X64-SSE4-NEXT: psrlw $4, %xmm8
-; X64-SSE4-NEXT: pand %xmm10, %xmm8
+; X64-SSE4-NEXT: pand %xmm11, %xmm8
; X64-SSE4-NEXT: movdqa %xmm9, %xmm0
; X64-SSE4-NEXT: pshufb %xmm8, %xmm0
; X64-SSE4-NEXT: paddb %xmm1, %xmm0
-; X64-SSE4-NEXT: paddb %xmm12, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm10, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm12, %xmm0
; X64-SSE4-NEXT: movdqa %xmm3, %xmm1
-; X64-SSE4-NEXT: pand %xmm10, %xmm1
+; X64-SSE4-NEXT: pand %xmm11, %xmm1
; X64-SSE4-NEXT: movdqa %xmm9, %xmm8
; X64-SSE4-NEXT: pshufb %xmm1, %xmm8
; X64-SSE4-NEXT: psrlw $4, %xmm3
-; X64-SSE4-NEXT: pand %xmm10, %xmm3
+; X64-SSE4-NEXT: pand %xmm11, %xmm3
; X64-SSE4-NEXT: movdqa %xmm9, %xmm1
; X64-SSE4-NEXT: pshufb %xmm3, %xmm1
; X64-SSE4-NEXT: paddb %xmm8, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm10, %xmm1
; X64-SSE4-NEXT: movdqa %xmm2, %xmm3
-; X64-SSE4-NEXT: pand %xmm10, %xmm3
+; X64-SSE4-NEXT: pand %xmm11, %xmm3
; X64-SSE4-NEXT: movdqa %xmm9, %xmm8
; X64-SSE4-NEXT: pshufb %xmm3, %xmm8
; X64-SSE4-NEXT: psrlw $4, %xmm2
-; X64-SSE4-NEXT: pand %xmm10, %xmm2
+; X64-SSE4-NEXT: pand %xmm11, %xmm2
; X64-SSE4-NEXT: movdqa %xmm9, %xmm3
; X64-SSE4-NEXT: pshufb %xmm2, %xmm3
; X64-SSE4-NEXT: paddb %xmm8, %xmm3
-; X64-SSE4-NEXT: paddb %xmm1, %xmm3
+; X64-SSE4-NEXT: psadbw %xmm10, %xmm3
+; X64-SSE4-NEXT: packusdw %xmm1, %xmm3
; X64-SSE4-NEXT: movdqa %xmm5, %xmm1
-; X64-SSE4-NEXT: pand %xmm10, %xmm1
+; X64-SSE4-NEXT: pand %xmm11, %xmm1
; X64-SSE4-NEXT: movdqa %xmm9, %xmm2
; X64-SSE4-NEXT: pshufb %xmm1, %xmm2
; X64-SSE4-NEXT: psrlw $4, %xmm5
-; X64-SSE4-NEXT: pand %xmm10, %xmm5
+; X64-SSE4-NEXT: pand %xmm11, %xmm5
; X64-SSE4-NEXT: movdqa %xmm9, %xmm8
; X64-SSE4-NEXT: pshufb %xmm5, %xmm8
; X64-SSE4-NEXT: paddb %xmm2, %xmm8
+; X64-SSE4-NEXT: psadbw %xmm10, %xmm8
; X64-SSE4-NEXT: movdqa %xmm4, %xmm1
-; X64-SSE4-NEXT: pand %xmm10, %xmm1
+; X64-SSE4-NEXT: pand %xmm11, %xmm1
; X64-SSE4-NEXT: movdqa %xmm9, %xmm2
; X64-SSE4-NEXT: pshufb %xmm1, %xmm2
; X64-SSE4-NEXT: psrlw $4, %xmm4
-; X64-SSE4-NEXT: pand %xmm10, %xmm4
+; X64-SSE4-NEXT: pand %xmm11, %xmm4
; X64-SSE4-NEXT: movdqa %xmm9, %xmm1
; X64-SSE4-NEXT: pshufb %xmm4, %xmm1
; X64-SSE4-NEXT: paddb %xmm2, %xmm1
-; X64-SSE4-NEXT: paddb %xmm8, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm10, %xmm1
+; X64-SSE4-NEXT: packusdw %xmm8, %xmm1
; X64-SSE4-NEXT: movdqa %xmm7, %xmm2
-; X64-SSE4-NEXT: pand %xmm10, %xmm2
+; X64-SSE4-NEXT: pand %xmm11, %xmm2
; X64-SSE4-NEXT: movdqa %xmm9, %xmm4
; X64-SSE4-NEXT: pshufb %xmm2, %xmm4
; X64-SSE4-NEXT: psrlw $4, %xmm7
-; X64-SSE4-NEXT: pand %xmm10, %xmm7
+; X64-SSE4-NEXT: pand %xmm11, %xmm7
; X64-SSE4-NEXT: movdqa %xmm9, %xmm2
; X64-SSE4-NEXT: pshufb %xmm7, %xmm2
; X64-SSE4-NEXT: paddb %xmm4, %xmm2
+; X64-SSE4-NEXT: psadbw %xmm10, %xmm2
; X64-SSE4-NEXT: movdqa %xmm6, %xmm4
-; X64-SSE4-NEXT: pand %xmm10, %xmm4
+; X64-SSE4-NEXT: pand %xmm11, %xmm4
; X64-SSE4-NEXT: movdqa %xmm9, %xmm5
; X64-SSE4-NEXT: pshufb %xmm4, %xmm5
; X64-SSE4-NEXT: psrlw $4, %xmm6
-; X64-SSE4-NEXT: pand %xmm10, %xmm6
+; X64-SSE4-NEXT: pand %xmm11, %xmm6
; X64-SSE4-NEXT: pshufb %xmm6, %xmm9
; X64-SSE4-NEXT: paddb %xmm5, %xmm9
-; X64-SSE4-NEXT: paddb %xmm2, %xmm9
-; X64-SSE4-NEXT: pxor %xmm2, %xmm2
-; X64-SSE4-NEXT: psadbw %xmm2, %xmm0
-; X64-SSE4-NEXT: psadbw %xmm2, %xmm3
-; X64-SSE4-NEXT: psadbw %xmm2, %xmm1
-; X64-SSE4-NEXT: psadbw %xmm2, %xmm9
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE4-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm3[1]
-; X64-SSE4-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm3[0]
-; X64-SSE4-NEXT: paddq %xmm2, %xmm0
-; X64-SSE4-NEXT: movdqa %xmm1, %xmm2
-; X64-SSE4-NEXT: punpckhqdq {{.*#+}} xmm2 = xmm2[1],xmm9[1]
-; X64-SSE4-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm9[0]
-; X64-SSE4-NEXT: paddq %xmm2, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm10, %xmm9
+; X64-SSE4-NEXT: packusdw %xmm2, %xmm9
+; X64-SSE4-NEXT: packusdw %xmm10, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm10, %xmm3
+; X64-SSE4-NEXT: packuswb %xmm3, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm10, %xmm1
+; X64-SSE4-NEXT: packusdw %xmm10, %xmm9
+; X64-SSE4-NEXT: packuswb %xmm9, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm10, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm10, %xmm1
; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: reduce_ctpop_v4i64_buildvector_v4i64:
@@ -3381,153 +3651,179 @@ define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64>
; X86-AVX1-NEXT: movl %esp, %ebp
; X86-AVX1-NEXT: andl $-32, %esp
; X86-AVX1-NEXT: subl $32, %esp
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5
-; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm6
-; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
-; X86-AVX1-NEXT: vpsrlw $4, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm6
-; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX1-NEXT: vpand %xmm4, %xmm3, %xmm6
+; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpshufb %xmm3, %xmm5, %xmm3
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm3, %xmm6
+; X86-AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
-; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm6
-; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
-; X86-AVX1-NEXT: vpsrlw $4, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm6
-; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
-; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm1, %xmm1
-; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5
-; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm6
-; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
-; X86-AVX1-NEXT: vpsrlw $4, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm6
-; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
-; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm2, %xmm2
-; X86-AVX1-NEXT: vmovdqa 8(%ebp), %xmm5
-; X86-AVX1-NEXT: vmovdqa 24(%ebp), %xmm6
-; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm7
-; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
-; X86-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
-; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm6
-; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
-; X86-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
-; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm7
-; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
-; X86-AVX1-NEXT: vpsrlw $4, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm3
-; X86-AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
-; X86-AVX1-NEXT: vpaddb %xmm7, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpunpckhqdq {{.*#+}} xmm5 = xmm2[1],xmm3[1]
-; X86-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X86-AVX1-NEXT: vpaddq %xmm5, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpunpckhqdq {{.*#+}} xmm3 = xmm0[1],xmm1[1]
-; X86-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
-; X86-AVX1-NEXT: movl %ebp, %esp
-; X86-AVX1-NEXT: popl %ebp
-; X86-AVX1-NEXT: retl
-;
-; X64-AVX1-LABEL: reduce_ctpop_v4i64_buildvector_v4i64:
-; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
-; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm7
-; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
-; X64-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm7
-; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
-; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm6
-; X64-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm7
-; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
-; X64-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm7
-; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
-; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm6, %xmm1, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6
-; X64-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm7
-; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
-; X64-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm7
-; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
-; X64-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpaddb %xmm6, %xmm2, %xmm2
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm6
-; X64-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm7
-; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
-; X64-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpand %xmm4, %xmm3, %xmm7
-; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
-; X64-AVX1-NEXT: vpsrlw $4, %xmm3, %xmm3
-; X64-AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3
-; X64-AVX1-NEXT: vpshufb %xmm3, %xmm5, %xmm3
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm3, %xmm3
-; X64-AVX1-NEXT: vpaddb %xmm6, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm6, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovdqa %xmm0, (%esp) # 16-byte Spill
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
+; X86-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
+; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm6, %xmm1, %xmm1
+; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
+; X86-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
+; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpackusdw %xmm6, %xmm2, %xmm2
+; X86-AVX1-NEXT: vmovdqa 24(%ebp), %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
+; X86-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
+; X86-AVX1-NEXT: vmovdqa 8(%ebp), %xmm7
+; X86-AVX1-NEXT: vpand %xmm4, %xmm7, %xmm0
+; X86-AVX1-NEXT: vpsrlw $4, %xmm7, %xmm7
+; X86-AVX1-NEXT: vpand %xmm4, %xmm7, %xmm4
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm4
+; X86-AVX1-NEXT: vpaddb %xmm0, %xmm4, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm6, %xmm4
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovdqa (%esp), %xmm4 # 16-byte Reload
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; X86-AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm4, %ymm1
+; X86-AVX1-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm1[0],ymm0[0],ymm1[2],ymm0[2]
+; X86-AVX1-NEXT: movl %ebp, %esp
+; X86-AVX1-NEXT: popl %ebp
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: reduce_ctpop_v4i64_buildvector_v4i64:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX1-NEXT: vpand %xmm5, %xmm4, %xmm7
+; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX1-NEXT: vpshufb %xmm7, %xmm6, %xmm7
+; X64-AVX1-NEXT: vpsrlw $4, %xmm4, %xmm4
+; X64-AVX1-NEXT: vpand %xmm5, %xmm4, %xmm4
+; X64-AVX1-NEXT: vpshufb %xmm4, %xmm6, %xmm4
+; X64-AVX1-NEXT: vpaddb %xmm7, %xmm4, %xmm7
; X64-AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpand %xmm5, %xmm0, %xmm8
+; X64-AVX1-NEXT: vpshufb %xmm8, %xmm6, %xmm8
+; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpand %xmm5, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufb %xmm0, %xmm6, %xmm0
+; X64-AVX1-NEXT: vpaddb %xmm0, %xmm8, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm7, %xmm0, %xmm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm7
+; X64-AVX1-NEXT: vpand %xmm5, %xmm7, %xmm8
+; X64-AVX1-NEXT: vpshufb %xmm8, %xmm6, %xmm8
+; X64-AVX1-NEXT: vpsrlw $4, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpand %xmm5, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpshufb %xmm7, %xmm6, %xmm7
+; X64-AVX1-NEXT: vpaddb %xmm7, %xmm8, %xmm7
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpand %xmm5, %xmm1, %xmm8
+; X64-AVX1-NEXT: vpshufb %xmm8, %xmm6, %xmm8
+; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpand %xmm5, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpshufb %xmm1, %xmm6, %xmm1
+; X64-AVX1-NEXT: vpaddb %xmm1, %xmm8, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm7, %xmm1, %xmm1
+; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm7
+; X64-AVX1-NEXT: vpand %xmm5, %xmm7, %xmm8
+; X64-AVX1-NEXT: vpshufb %xmm8, %xmm6, %xmm8
+; X64-AVX1-NEXT: vpsrlw $4, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpand %xmm5, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpshufb %xmm7, %xmm6, %xmm7
+; X64-AVX1-NEXT: vpaddb %xmm7, %xmm8, %xmm7
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpand %xmm5, %xmm2, %xmm8
+; X64-AVX1-NEXT: vpshufb %xmm8, %xmm6, %xmm8
+; X64-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpand %xmm5, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpshufb %xmm2, %xmm6, %xmm2
+; X64-AVX1-NEXT: vpaddb %xmm2, %xmm8, %xmm2
; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpackusdw %xmm7, %xmm2, %xmm2
+; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm7
+; X64-AVX1-NEXT: vpand %xmm5, %xmm7, %xmm8
+; X64-AVX1-NEXT: vpshufb %xmm8, %xmm6, %xmm8
+; X64-AVX1-NEXT: vpsrlw $4, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpand %xmm5, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpshufb %xmm7, %xmm6, %xmm7
+; X64-AVX1-NEXT: vpaddb %xmm7, %xmm8, %xmm7
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpand %xmm5, %xmm3, %xmm8
+; X64-AVX1-NEXT: vpshufb %xmm8, %xmm6, %xmm8
+; X64-AVX1-NEXT: vpsrlw $4, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpand %xmm5, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpshufb %xmm3, %xmm6, %xmm3
+; X64-AVX1-NEXT: vpaddb %xmm3, %xmm8, %xmm3
; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
-; X64-AVX1-NEXT: vpunpckhqdq {{.*#+}} xmm5 = xmm2[1],xmm3[1]
-; X64-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X64-AVX1-NEXT: vpaddq %xmm5, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm7, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpackusdw %xmm4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpunpckhqdq {{.*#+}} xmm3 = xmm0[1],xmm1[1]
-; X64-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X64-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
+; X64-AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
; X64-AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X64-AVX1-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: reduce_ctpop_v4i64_buildvector_v4i64:
@@ -3545,39 +3841,50 @@ define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64>
; X86-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
; X86-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
; X86-AVX2-NEXT: vpshufb %ymm0, %ymm5, %ymm0
-; X86-AVX2-NEXT: vpaddb %ymm6, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpxor %xmm6, %xmm6, %xmm6
-; X86-AVX2-NEXT: vpsadbw %ymm6, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpaddb %ymm6, %ymm0, %ymm6
+; X86-AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpsadbw %ymm0, %ymm6, %ymm6
; X86-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm7
; X86-AVX2-NEXT: vpshufb %ymm7, %ymm5, %ymm7
; X86-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
; X86-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
; X86-AVX2-NEXT: vpshufb %ymm1, %ymm5, %ymm1
; X86-AVX2-NEXT: vpaddb %ymm7, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpsadbw %ymm6, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpsadbw %ymm0, %ymm1, %ymm1
; X86-AVX2-NEXT: vpand %ymm4, %ymm2, %ymm7
; X86-AVX2-NEXT: vpshufb %ymm7, %ymm5, %ymm7
; X86-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
; X86-AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2
; X86-AVX2-NEXT: vpshufb %ymm2, %ymm5, %ymm2
; X86-AVX2-NEXT: vpaddb %ymm7, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpsadbw %ymm6, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpsadbw %ymm0, %ymm2, %ymm2
; X86-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm7
; X86-AVX2-NEXT: vpshufb %ymm7, %ymm5, %ymm7
; X86-AVX2-NEXT: vpsrlw $4, %ymm3, %ymm3
; X86-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
; X86-AVX2-NEXT: vpshufb %ymm3, %ymm5, %ymm3
; X86-AVX2-NEXT: vpaddb %ymm7, %ymm3, %ymm3
-; X86-AVX2-NEXT: vpsadbw %ymm6, %ymm3, %ymm3
-; X86-AVX2-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm1[2,3],ymm3[2,3]
-; X86-AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpaddq %ymm4, %ymm1, %ymm1
-; X86-AVX2-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[2,3],ymm2[2,3]
-; X86-AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpaddq %ymm3, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; X86-AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; X86-AVX2-NEXT: vpaddq %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpsadbw %ymm0, %ymm3, %ymm3
+; X86-AVX2-NEXT: vpackusdw %ymm0, %ymm6, %ymm4
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm0, %ymm4, %ymm4
+; X86-AVX2-NEXT: vpackusdw %ymm0, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm0, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpackuswb %ymm1, %ymm4, %ymm1
+; X86-AVX2-NEXT: vpackusdw %ymm0, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm0, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpackusdw %ymm0, %ymm3, %ymm3
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm0, %ymm3, %ymm3
+; X86-AVX2-NEXT: vpackuswb %ymm3, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,4,1,5,2,6,3,7]
+; X86-AVX2-NEXT: vpermd %ymm2, %ymm3, %ymm2
+; X86-AVX2-NEXT: vpermd %ymm1, %ymm3, %ymm1
+; X86-AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
+; X86-AVX2-NEXT: vpsadbw %ymm0, %ymm1, %ymm0
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; X86-AVX2-NEXT: movl %ebp, %esp
; X86-AVX2-NEXT: popl %ebp
; X86-AVX2-NEXT: retl
@@ -3592,39 +3899,50 @@ define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64>
; X64-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
; X64-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
; X64-AVX2-NEXT: vpshufb %ymm0, %ymm6, %ymm0
-; X64-AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; X64-AVX2-NEXT: vpsadbw %ymm5, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpaddb %ymm5, %ymm0, %ymm5
+; X64-AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm5, %ymm5
; X64-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm7
; X64-AVX2-NEXT: vpshufb %ymm7, %ymm6, %ymm7
; X64-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
; X64-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
; X64-AVX2-NEXT: vpshufb %ymm1, %ymm6, %ymm1
; X64-AVX2-NEXT: vpaddb %ymm7, %ymm1, %ymm1
-; X64-AVX2-NEXT: vpsadbw %ymm5, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm1, %ymm1
; X64-AVX2-NEXT: vpand %ymm4, %ymm2, %ymm7
; X64-AVX2-NEXT: vpshufb %ymm7, %ymm6, %ymm7
; X64-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
; X64-AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2
; X64-AVX2-NEXT: vpshufb %ymm2, %ymm6, %ymm2
; X64-AVX2-NEXT: vpaddb %ymm7, %ymm2, %ymm2
-; X64-AVX2-NEXT: vpsadbw %ymm5, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm2, %ymm2
; X64-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm7
; X64-AVX2-NEXT: vpshufb %ymm7, %ymm6, %ymm7
; X64-AVX2-NEXT: vpsrlw $4, %ymm3, %ymm3
; X64-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm3
; X64-AVX2-NEXT: vpshufb %ymm3, %ymm6, %ymm3
; X64-AVX2-NEXT: vpaddb %ymm7, %ymm3, %ymm3
-; X64-AVX2-NEXT: vpsadbw %ymm5, %ymm3, %ymm3
-; X64-AVX2-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm1[2,3],ymm3[2,3]
-; X64-AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; X64-AVX2-NEXT: vpaddq %ymm4, %ymm1, %ymm1
-; X64-AVX2-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[2,3],ymm2[2,3]
-; X64-AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpaddq %ymm3, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; X64-AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; X64-AVX2-NEXT: vpaddq %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm3, %ymm3
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm5, %ymm4
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm4, %ymm4
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpackuswb %ymm1, %ymm4, %ymm1
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm3, %ymm3
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm3, %ymm3
+; X64-AVX2-NEXT: vpackuswb %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpmovsxbd {{.*#+}} ymm3 = [0,4,1,5,2,6,3,7]
+; X64-AVX2-NEXT: vpermd %ymm2, %ymm3, %ymm2
+; X64-AVX2-NEXT: vpermd %ymm1, %ymm3, %ymm1
+; X64-AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm1 = ymm1[0],ymm2[0],ymm1[2],ymm2[2]
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; X64-AVX2-NEXT: retq
;
; AVX512VL-LABEL: reduce_ctpop_v4i64_buildvector_v4i64:
@@ -3661,15 +3979,15 @@ define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64>
; AVX512VL-NEXT: vpshufb %ymm3, %ymm6, %ymm3
; AVX512VL-NEXT: vpaddb %ymm7, %ymm3, %ymm3
; AVX512VL-NEXT: vpsadbw %ymm5, %ymm3, %ymm3
-; AVX512VL-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm1[2,3],ymm3[2,3]
-; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; AVX512VL-NEXT: vpaddq %ymm4, %ymm1, %ymm1
-; AVX512VL-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[2,3],ymm2[2,3]
-; AVX512VL-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX512VL-NEXT: vpaddq %ymm3, %ymm0, %ymm0
-; AVX512VL-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512VL-NEXT: vpaddq %ymm2, %ymm0, %ymm0
+; AVX512VL-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512VL-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512VL-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512VL-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
+; AVX512VL-NEXT: vpsadbw %ymm5, %ymm0, %ymm0
+; AVX512VL-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX512VL-NEXT: retq
;
; AVX512VPOPCNT-LABEL: reduce_ctpop_v4i64_buildvector_v4i64:
@@ -3678,15 +3996,16 @@ define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64>
; AVX512VPOPCNT-NEXT: vpopcntq %ymm1, %ymm1
; AVX512VPOPCNT-NEXT: vpopcntq %ymm2, %ymm2
; AVX512VPOPCNT-NEXT: vpopcntq %ymm3, %ymm3
-; AVX512VPOPCNT-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm1[2,3],ymm3[2,3]
-; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; AVX512VPOPCNT-NEXT: vpaddq %ymm4, %ymm1, %ymm1
-; AVX512VPOPCNT-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[2,3],ymm2[2,3]
-; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; AVX512VPOPCNT-NEXT: vpaddq %ymm3, %ymm0, %ymm0
-; AVX512VPOPCNT-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; AVX512VPOPCNT-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; AVX512VPOPCNT-NEXT: vpaddq %ymm2, %ymm0, %ymm0
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VPOPCNT-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
+; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512VPOPCNT-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512VPOPCNT-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; AVX512VPOPCNT-NEXT: retq
%p0 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a0)
%p1 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a1)
@@ -3709,551 +4028,591 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; X86-SSE2-NEXT: pushl %ebp
; X86-SSE2-NEXT: movl %esp, %ebp
; X86-SSE2-NEXT: andl $-16, %esp
-; X86-SSE2-NEXT: subl $80, %esp
-; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm6
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm5
+; X86-SSE2-NEXT: subl $64, %esp
+; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm4
; X86-SSE2-NEXT: movdqa %xmm1, %xmm3
; X86-SSE2-NEXT: psrlw $1, %xmm3
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm7 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; X86-SSE2-NEXT: pand %xmm7, %xmm3
; X86-SSE2-NEXT: psubb %xmm3, %xmm1
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
; X86-SSE2-NEXT: movdqa %xmm1, %xmm3
-; X86-SSE2-NEXT: pand %xmm4, %xmm3
+; X86-SSE2-NEXT: pand %xmm5, %xmm3
; X86-SSE2-NEXT: psrlw $2, %xmm1
-; X86-SSE2-NEXT: pand %xmm4, %xmm1
-; X86-SSE2-NEXT: paddb %xmm1, %xmm3
-; X86-SSE2-NEXT: movdqa %xmm3, %xmm1
-; X86-SSE2-NEXT: psrlw $4, %xmm1
+; X86-SSE2-NEXT: pand %xmm5, %xmm1
; X86-SSE2-NEXT: paddb %xmm3, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
-; X86-SSE2-NEXT: psrlw $1, %xmm3
-; X86-SSE2-NEXT: pand %xmm7, %xmm3
-; X86-SSE2-NEXT: psubb %xmm3, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
-; X86-SSE2-NEXT: pand %xmm4, %xmm3
-; X86-SSE2-NEXT: psrlw $2, %xmm0
-; X86-SSE2-NEXT: pand %xmm4, %xmm0
-; X86-SSE2-NEXT: paddb %xmm3, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm3
; X86-SSE2-NEXT: psrlw $4, %xmm3
-; X86-SSE2-NEXT: paddb %xmm3, %xmm0
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: paddb %xmm1, %xmm3
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm6 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm6, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT: psrlw $1, %xmm1
; X86-SSE2-NEXT: pand %xmm7, %xmm1
-; X86-SSE2-NEXT: pand %xmm7, %xmm0
+; X86-SSE2-NEXT: psubb %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT: pand %xmm5, %xmm1
+; X86-SSE2-NEXT: psrlw $2, %xmm0
+; X86-SSE2-NEXT: pand %xmm5, %xmm0
+; X86-SSE2-NEXT: paddb %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT: psrlw $4, %xmm1
; X86-SSE2-NEXT: paddb %xmm1, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm3
+; X86-SSE2-NEXT: pand %xmm6, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm3, %xmm0
; X86-SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm1
; X86-SSE2-NEXT: psrlw $1, %xmm1
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm0 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; X86-SSE2-NEXT: pand %xmm0, %xmm1
-; X86-SSE2-NEXT: psubb %xmm1, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
-; X86-SSE2-NEXT: pand %xmm4, %xmm1
-; X86-SSE2-NEXT: psrlw $2, %xmm5
-; X86-SSE2-NEXT: pand %xmm4, %xmm5
-; X86-SSE2-NEXT: paddb %xmm1, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
+; X86-SSE2-NEXT: pand %xmm7, %xmm1
+; X86-SSE2-NEXT: psubb %xmm1, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm1
+; X86-SSE2-NEXT: pand %xmm5, %xmm1
+; X86-SSE2-NEXT: psrlw $2, %xmm4
+; X86-SSE2-NEXT: pand %xmm5, %xmm4
+; X86-SSE2-NEXT: paddb %xmm1, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm1
; X86-SSE2-NEXT: psrlw $4, %xmm1
-; X86-SSE2-NEXT: paddb %xmm5, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
-; X86-SSE2-NEXT: psrlw $1, %xmm3
-; X86-SSE2-NEXT: pand %xmm0, %xmm3
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
-; X86-SSE2-NEXT: psubb %xmm3, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
-; X86-SSE2-NEXT: pand %xmm4, %xmm3
+; X86-SSE2-NEXT: paddb %xmm4, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: psrlw $1, %xmm0
+; X86-SSE2-NEXT: pand %xmm7, %xmm0
+; X86-SSE2-NEXT: psubb %xmm0, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: pand %xmm5, %xmm0
; X86-SSE2-NEXT: psrlw $2, %xmm2
-; X86-SSE2-NEXT: pand %xmm4, %xmm2
-; X86-SSE2-NEXT: paddb %xmm3, %xmm2
+; X86-SSE2-NEXT: pand %xmm5, %xmm2
+; X86-SSE2-NEXT: paddb %xmm0, %xmm2
; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
; X86-SSE2-NEXT: psrlw $4, %xmm3
; X86-SSE2-NEXT: paddb %xmm2, %xmm3
; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm2
-; X86-SSE2-NEXT: pand %xmm7, %xmm1
-; X86-SSE2-NEXT: pand %xmm7, %xmm3
-; X86-SSE2-NEXT: paddb %xmm1, %xmm3
-; X86-SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm1
-; X86-SSE2-NEXT: psrlw $1, %xmm1
-; X86-SSE2-NEXT: pand %xmm0, %xmm1
-; X86-SSE2-NEXT: psubb %xmm1, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm1
-; X86-SSE2-NEXT: pand %xmm4, %xmm1
-; X86-SSE2-NEXT: psrlw $2, %xmm2
-; X86-SSE2-NEXT: pand %xmm4, %xmm2
-; X86-SSE2-NEXT: paddb %xmm1, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm1
-; X86-SSE2-NEXT: psrlw $4, %xmm1
-; X86-SSE2-NEXT: paddb %xmm2, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm2
-; X86-SSE2-NEXT: psrlw $1, %xmm2
-; X86-SSE2-NEXT: pand %xmm0, %xmm2
-; X86-SSE2-NEXT: psubb %xmm2, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm2
-; X86-SSE2-NEXT: pand %xmm4, %xmm2
-; X86-SSE2-NEXT: psrlw $2, %xmm6
-; X86-SSE2-NEXT: pand %xmm4, %xmm6
-; X86-SSE2-NEXT: paddb %xmm2, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm3
-; X86-SSE2-NEXT: psrlw $4, %xmm3
-; X86-SSE2-NEXT: paddb %xmm6, %xmm3
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm2
-; X86-SSE2-NEXT: pand %xmm7, %xmm1
-; X86-SSE2-NEXT: pand %xmm7, %xmm3
-; X86-SSE2-NEXT: movdqa %xmm7, %xmm6
-; X86-SSE2-NEXT: paddb %xmm1, %xmm3
+; X86-SSE2-NEXT: pand %xmm6, %xmm1
+; X86-SSE2-NEXT: pxor %xmm0, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE2-NEXT: pand %xmm6, %xmm3
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm1, %xmm3
; X86-SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
; X86-SSE2-NEXT: psrlw $1, %xmm0
-; X86-SSE2-NEXT: pand %xmm5, %xmm0
+; X86-SSE2-NEXT: pand %xmm7, %xmm0
; X86-SSE2-NEXT: psubb %xmm0, %xmm2
; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE2-NEXT: pand %xmm4, %xmm0
+; X86-SSE2-NEXT: pand %xmm5, %xmm0
; X86-SSE2-NEXT: psrlw $2, %xmm2
-; X86-SSE2-NEXT: pand %xmm4, %xmm2
+; X86-SSE2-NEXT: pand %xmm5, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm5, %xmm4
; X86-SSE2-NEXT: paddb %xmm0, %xmm2
; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
; X86-SSE2-NEXT: psrlw $4, %xmm0
; X86-SSE2-NEXT: paddb %xmm2, %xmm0
-; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm1
-; X86-SSE2-NEXT: psrlw $1, %xmm1
+; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT: psrlw $1, %xmm2
+; X86-SSE2-NEXT: pand %xmm7, %xmm2
+; X86-SSE2-NEXT: psubb %xmm2, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT: pand %xmm5, %xmm2
+; X86-SSE2-NEXT: psrlw $2, %xmm1
; X86-SSE2-NEXT: pand %xmm5, %xmm1
-; X86-SSE2-NEXT: psubb %xmm1, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm1
-; X86-SSE2-NEXT: pand %xmm4, %xmm1
-; X86-SSE2-NEXT: psrlw $2, %xmm2
-; X86-SSE2-NEXT: pand %xmm4, %xmm2
-; X86-SSE2-NEXT: paddb %xmm1, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm1
-; X86-SSE2-NEXT: psrlw $4, %xmm1
; X86-SSE2-NEXT: paddb %xmm2, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT: psrlw $4, %xmm2
+; X86-SSE2-NEXT: paddb %xmm1, %xmm2
+; X86-SSE2-NEXT: pand %xmm6, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: pand %xmm6, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm6, %xmm3
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm2
+; X86-SSE2-NEXT: packuswb %xmm0, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm2, (%esp) # 16-byte Spill
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: psrlw $1, %xmm0
; X86-SSE2-NEXT: pand %xmm7, %xmm0
-; X86-SSE2-NEXT: pand %xmm7, %xmm1
+; X86-SSE2-NEXT: psubb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: pand %xmm5, %xmm0
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm5, %xmm1
; X86-SSE2-NEXT: paddb %xmm0, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm1, (%esp) # 16-byte Spill
-; X86-SSE2-NEXT: movdqa 104(%ebp), %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: psrlw $4, %xmm0
+; X86-SSE2-NEXT: paddb %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT: psrlw $1, %xmm2
+; X86-SSE2-NEXT: pand %xmm7, %xmm2
+; X86-SSE2-NEXT: psubb %xmm2, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT: pand %xmm5, %xmm2
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm5, %xmm1
+; X86-SSE2-NEXT: paddb %xmm2, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm6
+; X86-SSE2-NEXT: psrlw $4, %xmm6
+; X86-SSE2-NEXT: paddb %xmm1, %xmm6
+; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: pand %xmm3, %xmm6
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm6
+; X86-SSE2-NEXT: packuswb %xmm0, %xmm6
+; X86-SSE2-NEXT: movdqa 104(%ebp), %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
; X86-SSE2-NEXT: psrlw $1, %xmm0
+; X86-SSE2-NEXT: pand %xmm7, %xmm0
+; X86-SSE2-NEXT: psubb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
; X86-SSE2-NEXT: pand %xmm5, %xmm0
-; X86-SSE2-NEXT: psubb %xmm0, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE2-NEXT: pand %xmm4, %xmm0
-; X86-SSE2-NEXT: psrlw $2, %xmm2
-; X86-SSE2-NEXT: pand %xmm4, %xmm2
-; X86-SSE2-NEXT: paddb %xmm0, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm5, %xmm1
+; X86-SSE2-NEXT: paddb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
; X86-SSE2-NEXT: psrlw $4, %xmm0
-; X86-SSE2-NEXT: paddb %xmm2, %xmm0
-; X86-SSE2-NEXT: movdqa 88(%ebp), %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
-; X86-SSE2-NEXT: psrlw $1, %xmm3
-; X86-SSE2-NEXT: pand %xmm5, %xmm3
-; X86-SSE2-NEXT: psubb %xmm3, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
-; X86-SSE2-NEXT: pand %xmm4, %xmm3
-; X86-SSE2-NEXT: psrlw $2, %xmm2
-; X86-SSE2-NEXT: pand %xmm4, %xmm2
-; X86-SSE2-NEXT: paddb %xmm3, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm7
-; X86-SSE2-NEXT: psrlw $4, %xmm7
-; X86-SSE2-NEXT: paddb %xmm2, %xmm7
-; X86-SSE2-NEXT: pand %xmm6, %xmm0
-; X86-SSE2-NEXT: pand %xmm6, %xmm7
-; X86-SSE2-NEXT: movdqa %xmm6, %xmm1
-; X86-SSE2-NEXT: paddb %xmm0, %xmm7
-; X86-SSE2-NEXT: movdqa 136(%ebp), %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: paddb %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa 88(%ebp), %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT: psrlw $1, %xmm2
+; X86-SSE2-NEXT: pand %xmm7, %xmm2
+; X86-SSE2-NEXT: psubb %xmm2, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT: pand %xmm5, %xmm2
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm5, %xmm1
+; X86-SSE2-NEXT: paddb %xmm2, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm5
+; X86-SSE2-NEXT: psrlw $4, %xmm5
+; X86-SSE2-NEXT: paddb %xmm1, %xmm5
+; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: pand %xmm3, %xmm5
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm5
+; X86-SSE2-NEXT: packuswb %xmm0, %xmm5
+; X86-SSE2-NEXT: movdqa 136(%ebp), %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
; X86-SSE2-NEXT: psrlw $1, %xmm0
-; X86-SSE2-NEXT: pand %xmm5, %xmm0
-; X86-SSE2-NEXT: psubb %xmm0, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm7, %xmm3
+; X86-SSE2-NEXT: pand %xmm7, %xmm0
+; X86-SSE2-NEXT: psubb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
; X86-SSE2-NEXT: pand %xmm4, %xmm0
-; X86-SSE2-NEXT: psrlw $2, %xmm2
-; X86-SSE2-NEXT: pand %xmm4, %xmm2
-; X86-SSE2-NEXT: paddb %xmm0, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm4, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm7
+; X86-SSE2-NEXT: paddb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
; X86-SSE2-NEXT: psrlw $4, %xmm0
-; X86-SSE2-NEXT: paddb %xmm2, %xmm0
-; X86-SSE2-NEXT: movdqa 120(%ebp), %xmm3
-; X86-SSE2-NEXT: movdqa %xmm3, %xmm2
+; X86-SSE2-NEXT: paddb %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa 120(%ebp), %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm2
; X86-SSE2-NEXT: psrlw $1, %xmm2
-; X86-SSE2-NEXT: pand %xmm5, %xmm2
-; X86-SSE2-NEXT: psubb %xmm2, %xmm3
-; X86-SSE2-NEXT: movdqa %xmm3, %xmm2
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: psubb %xmm2, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm2
; X86-SSE2-NEXT: pand %xmm4, %xmm2
-; X86-SSE2-NEXT: psrlw $2, %xmm3
-; X86-SSE2-NEXT: pand %xmm4, %xmm3
-; X86-SSE2-NEXT: paddb %xmm2, %xmm3
-; X86-SSE2-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE2-NEXT: psrlw $4, %xmm6
-; X86-SSE2-NEXT: paddb %xmm3, %xmm6
-; X86-SSE2-NEXT: pand %xmm1, %xmm0
-; X86-SSE2-NEXT: pand %xmm1, %xmm6
-; X86-SSE2-NEXT: paddb %xmm0, %xmm6
-; X86-SSE2-NEXT: movdqa 168(%ebp), %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm4, %xmm1
+; X86-SSE2-NEXT: paddb %xmm2, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
+; X86-SSE2-NEXT: psrlw $4, %xmm4
+; X86-SSE2-NEXT: paddb %xmm1, %xmm4
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm2, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: pand %xmm2, %xmm4
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm4
+; X86-SSE2-NEXT: packuswb %xmm0, %xmm4
+; X86-SSE2-NEXT: movdqa 168(%ebp), %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: psrlw $1, %xmm0
+; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: psubb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: pand %xmm7, %xmm0
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm7, %xmm1
+; X86-SSE2-NEXT: paddb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: psrlw $4, %xmm0
+; X86-SSE2-NEXT: paddb %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa 152(%ebp), %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm2
; X86-SSE2-NEXT: psrlw $1, %xmm2
-; X86-SSE2-NEXT: pand %xmm5, %xmm2
-; X86-SSE2-NEXT: psubb %xmm2, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm2
-; X86-SSE2-NEXT: pand %xmm4, %xmm2
-; X86-SSE2-NEXT: psrlw $2, %xmm0
-; X86-SSE2-NEXT: pand %xmm4, %xmm0
-; X86-SSE2-NEXT: paddb %xmm2, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm7
+; X86-SSE2-NEXT: psubb %xmm2, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm3, %xmm1
+; X86-SSE2-NEXT: paddb %xmm2, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm3
; X86-SSE2-NEXT: psrlw $4, %xmm3
-; X86-SSE2-NEXT: paddb %xmm0, %xmm3
-; X86-SSE2-NEXT: movdqa 152(%ebp), %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm2
-; X86-SSE2-NEXT: psrlw $1, %xmm2
-; X86-SSE2-NEXT: pand %xmm5, %xmm2
-; X86-SSE2-NEXT: psubb %xmm2, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm2
-; X86-SSE2-NEXT: pand %xmm4, %xmm2
+; X86-SSE2-NEXT: paddb %xmm1, %xmm3
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm1, %xmm0
+; X86-SSE2-NEXT: pxor %xmm2, %xmm2
+; X86-SSE2-NEXT: psadbw %xmm2, %xmm0
+; X86-SSE2-NEXT: pand %xmm1, %xmm3
+; X86-SSE2-NEXT: psadbw %xmm2, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm0, %xmm3
+; X86-SSE2-NEXT: movdqa 200(%ebp), %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT: psrlw $1, %xmm1
+; X86-SSE2-NEXT: pand %xmm7, %xmm1
+; X86-SSE2-NEXT: psubb %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm7 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; X86-SSE2-NEXT: pand %xmm7, %xmm1
; X86-SSE2-NEXT: psrlw $2, %xmm0
-; X86-SSE2-NEXT: pand %xmm4, %xmm0
-; X86-SSE2-NEXT: paddb %xmm2, %xmm0
+; X86-SSE2-NEXT: pand %xmm7, %xmm0
+; X86-SSE2-NEXT: paddb %xmm1, %xmm0
; X86-SSE2-NEXT: movdqa %xmm0, %xmm2
; X86-SSE2-NEXT: psrlw $4, %xmm2
; X86-SSE2-NEXT: paddb %xmm0, %xmm2
-; X86-SSE2-NEXT: pand %xmm1, %xmm3
-; X86-SSE2-NEXT: pand %xmm1, %xmm2
-; X86-SSE2-NEXT: paddb %xmm3, %xmm2
-; X86-SSE2-NEXT: movdqa 200(%ebp), %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
-; X86-SSE2-NEXT: psrlw $1, %xmm3
-; X86-SSE2-NEXT: pand %xmm5, %xmm3
-; X86-SSE2-NEXT: psubb %xmm3, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
-; X86-SSE2-NEXT: pand %xmm4, %xmm3
+; X86-SSE2-NEXT: movdqa 184(%ebp), %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT: psrlw $1, %xmm1
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE2-NEXT: psubb %xmm1, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT: pand %xmm7, %xmm1
; X86-SSE2-NEXT: psrlw $2, %xmm0
-; X86-SSE2-NEXT: pand %xmm4, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm4, %xmm5
-; X86-SSE2-NEXT: paddb %xmm3, %xmm0
+; X86-SSE2-NEXT: pand %xmm7, %xmm0
+; X86-SSE2-NEXT: paddb %xmm1, %xmm0
; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
; X86-SSE2-NEXT: psrlw $4, %xmm1
; X86-SSE2-NEXT: paddb %xmm0, %xmm1
-; X86-SSE2-NEXT: movdqa 184(%ebp), %xmm4
-; X86-SSE2-NEXT: movdqa %xmm4, %xmm0
-; X86-SSE2-NEXT: psrlw $1, %xmm0
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE2-NEXT: psubb %xmm0, %xmm4
-; X86-SSE2-NEXT: movdqa %xmm4, %xmm0
-; X86-SSE2-NEXT: pand %xmm5, %xmm0
-; X86-SSE2-NEXT: psrlw $2, %xmm4
-; X86-SSE2-NEXT: pand %xmm5, %xmm4
-; X86-SSE2-NEXT: paddb %xmm0, %xmm4
-; X86-SSE2-NEXT: movdqa %xmm4, %xmm3
-; X86-SSE2-NEXT: psrlw $4, %xmm3
-; X86-SSE2-NEXT: paddb %xmm4, %xmm3
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm0, %xmm2
; X86-SSE2-NEXT: pand %xmm0, %xmm1
-; X86-SSE2-NEXT: pand %xmm0, %xmm3
-; X86-SSE2-NEXT: paddb %xmm1, %xmm3
; X86-SSE2-NEXT: pxor %xmm0, %xmm0
-; X86-SSE2-NEXT: movdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm5 # 16-byte Reload
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm5
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm4, %xmm5
-; X86-SSE2-NEXT: movdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm2
; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm1, %xmm4
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm1
+; X86-SSE2-NEXT: movdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-SSE2-NEXT: pxor %xmm2, %xmm2
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm2, %xmm0
+; X86-SSE2-NEXT: movdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm7 # 16-byte Reload
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm7
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm7
+; X86-SSE2-NEXT: psadbw %xmm2, %xmm7
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]
+; X86-SSE2-NEXT: movdqa (%esp), %xmm7 # 16-byte Reload
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm7
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm7
+; X86-SSE2-NEXT: psadbw %xmm2, %xmm7
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm6
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm6
+; X86-SSE2-NEXT: psadbw %xmm2, %xmm6
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1]
+; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm7[0]
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm5
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm5
+; X86-SSE2-NEXT: psadbw %xmm2, %xmm5
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm4
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm4
+; X86-SSE2-NEXT: psadbw %xmm2, %xmm4
; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1]
-; X86-SSE2-NEXT: movdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm1, %xmm4
-; X86-SSE2-NEXT: movdqa (%esp), %xmm0 # 16-byte Reload
-; X86-SSE2-NEXT: pxor %xmm1, %xmm1
-; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm0, %xmm1
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
-; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm4[0]
-; X86-SSE2-NEXT: pxor %xmm0, %xmm0
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm7
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm7, %xmm1
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm6
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm6[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm6, %xmm4
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm2
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm2, %xmm4
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm3
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm3, %xmm2
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm0
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm3
+; X86-SSE2-NEXT: psadbw %xmm2, %xmm3
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm1
+; X86-SSE2-NEXT: packuswb %xmm2, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm2, %xmm1
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; X86-SSE2-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm3[0]
+; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
; X86-SSE2-NEXT: movl %ebp, %esp
; X86-SSE2-NEXT: popl %ebp
; X86-SSE2-NEXT: retl
;
; X64-SSE2-LABEL: reduce_ctpop_v4i64_buildvector_v8i32:
; X64-SSE2: # %bb.0:
-; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm8
-; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm11
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm9
-; X64-SSE2-NEXT: psrlw $1, %xmm9
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm10 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; X64-SSE2-NEXT: pand %xmm10, %xmm9
-; X64-SSE2-NEXT: psubb %xmm9, %xmm1
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm9 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm12
-; X64-SSE2-NEXT: pand %xmm9, %xmm12
+; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm9
+; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm13
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm8
+; X64-SSE2-NEXT: psrlw $1, %xmm8
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm11 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
+; X64-SSE2-NEXT: pand %xmm11, %xmm8
+; X64-SSE2-NEXT: psubb %xmm8, %xmm1
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm10 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm8
+; X64-SSE2-NEXT: pand %xmm10, %xmm8
; X64-SSE2-NEXT: psrlw $2, %xmm1
-; X64-SSE2-NEXT: pand %xmm9, %xmm1
-; X64-SSE2-NEXT: paddb %xmm1, %xmm12
-; X64-SSE2-NEXT: movdqa %xmm12, %xmm13
-; X64-SSE2-NEXT: psrlw $4, %xmm13
-; X64-SSE2-NEXT: paddb %xmm12, %xmm13
+; X64-SSE2-NEXT: pand %xmm10, %xmm1
+; X64-SSE2-NEXT: paddb %xmm8, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm14
+; X64-SSE2-NEXT: psrlw $4, %xmm14
+; X64-SSE2-NEXT: paddb %xmm1, %xmm14
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm12 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-SSE2-NEXT: pand %xmm12, %xmm14
; X64-SSE2-NEXT: movdqa %xmm0, %xmm1
; X64-SSE2-NEXT: psrlw $1, %xmm1
-; X64-SSE2-NEXT: pand %xmm10, %xmm1
+; X64-SSE2-NEXT: pand %xmm11, %xmm1
; X64-SSE2-NEXT: psubb %xmm1, %xmm0
; X64-SSE2-NEXT: movdqa %xmm0, %xmm1
-; X64-SSE2-NEXT: pand %xmm9, %xmm1
+; X64-SSE2-NEXT: pand %xmm10, %xmm1
; X64-SSE2-NEXT: psrlw $2, %xmm0
-; X64-SSE2-NEXT: pand %xmm9, %xmm0
+; X64-SSE2-NEXT: pand %xmm10, %xmm0
; X64-SSE2-NEXT: paddb %xmm1, %xmm0
; X64-SSE2-NEXT: movdqa %xmm0, %xmm1
; X64-SSE2-NEXT: psrlw $4, %xmm1
; X64-SSE2-NEXT: paddb %xmm1, %xmm0
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-SSE2-NEXT: pand %xmm1, %xmm13
-; X64-SSE2-NEXT: pand %xmm1, %xmm0
-; X64-SSE2-NEXT: paddb %xmm13, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm3, %xmm12
-; X64-SSE2-NEXT: psrlw $1, %xmm12
-; X64-SSE2-NEXT: pand %xmm10, %xmm12
-; X64-SSE2-NEXT: psubb %xmm12, %xmm3
-; X64-SSE2-NEXT: movdqa %xmm3, %xmm12
-; X64-SSE2-NEXT: pand %xmm9, %xmm12
+; X64-SSE2-NEXT: pxor %xmm8, %xmm8
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm14
+; X64-SSE2-NEXT: pand %xmm12, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm14, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
+; X64-SSE2-NEXT: psrlw $1, %xmm1
+; X64-SSE2-NEXT: pand %xmm11, %xmm1
+; X64-SSE2-NEXT: psubb %xmm1, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
+; X64-SSE2-NEXT: pand %xmm10, %xmm1
; X64-SSE2-NEXT: psrlw $2, %xmm3
-; X64-SSE2-NEXT: pand %xmm9, %xmm3
-; X64-SSE2-NEXT: paddb %xmm12, %xmm3
-; X64-SSE2-NEXT: movdqa %xmm3, %xmm13
-; X64-SSE2-NEXT: psrlw $4, %xmm13
-; X64-SSE2-NEXT: paddb %xmm3, %xmm13
+; X64-SSE2-NEXT: pand %xmm10, %xmm3
+; X64-SSE2-NEXT: paddb %xmm1, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
+; X64-SSE2-NEXT: psrlw $4, %xmm1
+; X64-SSE2-NEXT: paddb %xmm3, %xmm1
; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
; X64-SSE2-NEXT: psrlw $1, %xmm3
-; X64-SSE2-NEXT: pand %xmm10, %xmm3
+; X64-SSE2-NEXT: pand %xmm11, %xmm3
; X64-SSE2-NEXT: psubb %xmm3, %xmm2
; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
-; X64-SSE2-NEXT: pand %xmm9, %xmm3
+; X64-SSE2-NEXT: pand %xmm10, %xmm3
; X64-SSE2-NEXT: psrlw $2, %xmm2
-; X64-SSE2-NEXT: pand %xmm9, %xmm2
+; X64-SSE2-NEXT: pand %xmm10, %xmm2
; X64-SSE2-NEXT: paddb %xmm3, %xmm2
; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
; X64-SSE2-NEXT: psrlw $4, %xmm3
; X64-SSE2-NEXT: paddb %xmm2, %xmm3
-; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm12
-; X64-SSE2-NEXT: pand %xmm1, %xmm13
-; X64-SSE2-NEXT: pand %xmm1, %xmm3
-; X64-SSE2-NEXT: paddb %xmm13, %xmm3
-; X64-SSE2-NEXT: movdqa %xmm5, %xmm2
-; X64-SSE2-NEXT: psrlw $1, %xmm2
-; X64-SSE2-NEXT: pand %xmm10, %xmm2
-; X64-SSE2-NEXT: psubb %xmm2, %xmm5
-; X64-SSE2-NEXT: movdqa %xmm5, %xmm2
-; X64-SSE2-NEXT: pand %xmm9, %xmm2
+; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm14
+; X64-SSE2-NEXT: pand %xmm12, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm1
+; X64-SSE2-NEXT: pand %xmm12, %xmm3
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm1
+; X64-SSE2-NEXT: psrlw $1, %xmm1
+; X64-SSE2-NEXT: pand %xmm11, %xmm1
+; X64-SSE2-NEXT: psubb %xmm1, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm1
+; X64-SSE2-NEXT: pand %xmm10, %xmm1
; X64-SSE2-NEXT: psrlw $2, %xmm5
-; X64-SSE2-NEXT: pand %xmm9, %xmm5
-; X64-SSE2-NEXT: paddb %xmm2, %xmm5
-; X64-SSE2-NEXT: movdqa %xmm5, %xmm14
-; X64-SSE2-NEXT: psrlw $4, %xmm14
-; X64-SSE2-NEXT: paddb %xmm5, %xmm14
+; X64-SSE2-NEXT: pand %xmm10, %xmm5
+; X64-SSE2-NEXT: paddb %xmm1, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm1
+; X64-SSE2-NEXT: psrlw $4, %xmm1
+; X64-SSE2-NEXT: paddb %xmm5, %xmm1
; X64-SSE2-NEXT: movdqa %xmm4, %xmm2
; X64-SSE2-NEXT: psrlw $1, %xmm2
-; X64-SSE2-NEXT: pand %xmm10, %xmm2
+; X64-SSE2-NEXT: pand %xmm11, %xmm2
; X64-SSE2-NEXT: psubb %xmm2, %xmm4
; X64-SSE2-NEXT: movdqa %xmm4, %xmm2
-; X64-SSE2-NEXT: pand %xmm9, %xmm2
+; X64-SSE2-NEXT: pand %xmm10, %xmm2
; X64-SSE2-NEXT: psrlw $2, %xmm4
-; X64-SSE2-NEXT: pand %xmm9, %xmm4
+; X64-SSE2-NEXT: pand %xmm10, %xmm4
; X64-SSE2-NEXT: paddb %xmm2, %xmm4
; X64-SSE2-NEXT: movdqa %xmm4, %xmm2
; X64-SSE2-NEXT: psrlw $4, %xmm2
; X64-SSE2-NEXT: paddb %xmm4, %xmm2
-; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm13
-; X64-SSE2-NEXT: pand %xmm1, %xmm14
-; X64-SSE2-NEXT: pand %xmm1, %xmm2
-; X64-SSE2-NEXT: paddb %xmm14, %xmm2
-; X64-SSE2-NEXT: movdqa %xmm7, %xmm4
-; X64-SSE2-NEXT: psrlw $1, %xmm4
-; X64-SSE2-NEXT: pand %xmm10, %xmm4
-; X64-SSE2-NEXT: psubb %xmm4, %xmm7
-; X64-SSE2-NEXT: movdqa %xmm7, %xmm4
-; X64-SSE2-NEXT: pand %xmm9, %xmm4
+; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm5
+; X64-SSE2-NEXT: pand %xmm12, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm1
+; X64-SSE2-NEXT: pand %xmm12, %xmm2
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm2
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm1
+; X64-SSE2-NEXT: psrlw $1, %xmm1
+; X64-SSE2-NEXT: pand %xmm11, %xmm1
+; X64-SSE2-NEXT: psubb %xmm1, %xmm7
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm1
+; X64-SSE2-NEXT: pand %xmm10, %xmm1
; X64-SSE2-NEXT: psrlw $2, %xmm7
-; X64-SSE2-NEXT: pand %xmm9, %xmm7
-; X64-SSE2-NEXT: paddb %xmm4, %xmm7
-; X64-SSE2-NEXT: movdqa %xmm7, %xmm5
-; X64-SSE2-NEXT: psrlw $4, %xmm5
-; X64-SSE2-NEXT: paddb %xmm7, %xmm5
+; X64-SSE2-NEXT: pand %xmm10, %xmm7
+; X64-SSE2-NEXT: paddb %xmm1, %xmm7
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm1
+; X64-SSE2-NEXT: psrlw $4, %xmm1
+; X64-SSE2-NEXT: paddb %xmm7, %xmm1
; X64-SSE2-NEXT: movdqa %xmm6, %xmm4
; X64-SSE2-NEXT: psrlw $1, %xmm4
-; X64-SSE2-NEXT: pand %xmm10, %xmm4
+; X64-SSE2-NEXT: pand %xmm11, %xmm4
; X64-SSE2-NEXT: psubb %xmm4, %xmm6
; X64-SSE2-NEXT: movdqa %xmm6, %xmm4
-; X64-SSE2-NEXT: pand %xmm9, %xmm4
+; X64-SSE2-NEXT: pand %xmm10, %xmm4
; X64-SSE2-NEXT: psrlw $2, %xmm6
-; X64-SSE2-NEXT: pand %xmm9, %xmm6
+; X64-SSE2-NEXT: pand %xmm10, %xmm6
; X64-SSE2-NEXT: paddb %xmm4, %xmm6
; X64-SSE2-NEXT: movdqa %xmm6, %xmm4
; X64-SSE2-NEXT: psrlw $4, %xmm4
; X64-SSE2-NEXT: paddb %xmm6, %xmm4
-; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm6
-; X64-SSE2-NEXT: pand %xmm1, %xmm5
-; X64-SSE2-NEXT: pand %xmm1, %xmm4
-; X64-SSE2-NEXT: paddb %xmm5, %xmm4
-; X64-SSE2-NEXT: movdqa %xmm6, %xmm5
-; X64-SSE2-NEXT: psrlw $1, %xmm5
+; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm7
+; X64-SSE2-NEXT: pand %xmm12, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm1
+; X64-SSE2-NEXT: pand %xmm12, %xmm4
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm4
+; X64-SSE2-NEXT: packuswb %xmm1, %xmm4
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm1
+; X64-SSE2-NEXT: psrlw $1, %xmm1
+; X64-SSE2-NEXT: pand %xmm11, %xmm1
+; X64-SSE2-NEXT: psubb %xmm1, %xmm7
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm1
+; X64-SSE2-NEXT: pand %xmm10, %xmm1
+; X64-SSE2-NEXT: psrlw $2, %xmm7
+; X64-SSE2-NEXT: pand %xmm10, %xmm7
+; X64-SSE2-NEXT: paddb %xmm1, %xmm7
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm6
+; X64-SSE2-NEXT: psrlw $4, %xmm6
+; X64-SSE2-NEXT: paddb %xmm7, %xmm6
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm1
+; X64-SSE2-NEXT: psrlw $1, %xmm1
+; X64-SSE2-NEXT: pand %xmm11, %xmm1
+; X64-SSE2-NEXT: psubb %xmm1, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm1
+; X64-SSE2-NEXT: pand %xmm10, %xmm1
+; X64-SSE2-NEXT: psrlw $2, %xmm5
; X64-SSE2-NEXT: pand %xmm10, %xmm5
-; X64-SSE2-NEXT: psubb %xmm5, %xmm6
-; X64-SSE2-NEXT: movdqa %xmm6, %xmm5
-; X64-SSE2-NEXT: pand %xmm9, %xmm5
-; X64-SSE2-NEXT: psrlw $2, %xmm6
-; X64-SSE2-NEXT: pand %xmm9, %xmm6
+; X64-SSE2-NEXT: paddb %xmm1, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm1
+; X64-SSE2-NEXT: psrlw $4, %xmm1
+; X64-SSE2-NEXT: paddb %xmm5, %xmm1
+; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm5
+; X64-SSE2-NEXT: pand %xmm12, %xmm6
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm6
+; X64-SSE2-NEXT: pand %xmm12, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm6, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm6
+; X64-SSE2-NEXT: psrlw $1, %xmm6
+; X64-SSE2-NEXT: pand %xmm11, %xmm6
+; X64-SSE2-NEXT: psubb %xmm6, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm6
+; X64-SSE2-NEXT: pand %xmm10, %xmm6
+; X64-SSE2-NEXT: psrlw $2, %xmm5
+; X64-SSE2-NEXT: pand %xmm10, %xmm5
+; X64-SSE2-NEXT: paddb %xmm6, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm6
+; X64-SSE2-NEXT: psrlw $4, %xmm6
; X64-SSE2-NEXT: paddb %xmm5, %xmm6
-; X64-SSE2-NEXT: movdqa %xmm6, %xmm7
-; X64-SSE2-NEXT: psrlw $4, %xmm7
-; X64-SSE2-NEXT: paddb %xmm6, %xmm7
-; X64-SSE2-NEXT: movdqa %xmm13, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm14, %xmm5
; X64-SSE2-NEXT: psrlw $1, %xmm5
+; X64-SSE2-NEXT: pand %xmm11, %xmm5
+; X64-SSE2-NEXT: psubb %xmm5, %xmm14
+; X64-SSE2-NEXT: movdqa %xmm14, %xmm5
; X64-SSE2-NEXT: pand %xmm10, %xmm5
-; X64-SSE2-NEXT: psubb %xmm5, %xmm13
-; X64-SSE2-NEXT: movdqa %xmm13, %xmm5
-; X64-SSE2-NEXT: pand %xmm9, %xmm5
-; X64-SSE2-NEXT: psrlw $2, %xmm13
-; X64-SSE2-NEXT: pand %xmm9, %xmm13
-; X64-SSE2-NEXT: paddb %xmm5, %xmm13
-; X64-SSE2-NEXT: movdqa %xmm13, %xmm5
+; X64-SSE2-NEXT: psrlw $2, %xmm14
+; X64-SSE2-NEXT: pand %xmm10, %xmm14
+; X64-SSE2-NEXT: paddb %xmm5, %xmm14
+; X64-SSE2-NEXT: movdqa %xmm14, %xmm5
; X64-SSE2-NEXT: psrlw $4, %xmm5
-; X64-SSE2-NEXT: paddb %xmm13, %xmm5
-; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm6
-; X64-SSE2-NEXT: pand %xmm1, %xmm7
-; X64-SSE2-NEXT: pand %xmm1, %xmm5
-; X64-SSE2-NEXT: paddb %xmm7, %xmm5
-; X64-SSE2-NEXT: movdqa %xmm6, %xmm7
-; X64-SSE2-NEXT: psrlw $1, %xmm7
-; X64-SSE2-NEXT: pand %xmm10, %xmm7
-; X64-SSE2-NEXT: psubb %xmm7, %xmm6
-; X64-SSE2-NEXT: movdqa %xmm6, %xmm7
-; X64-SSE2-NEXT: pand %xmm9, %xmm7
-; X64-SSE2-NEXT: psrlw $2, %xmm6
-; X64-SSE2-NEXT: pand %xmm9, %xmm6
-; X64-SSE2-NEXT: paddb %xmm7, %xmm6
-; X64-SSE2-NEXT: movdqa %xmm6, %xmm7
+; X64-SSE2-NEXT: paddb %xmm14, %xmm5
+; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm14
+; X64-SSE2-NEXT: pand %xmm12, %xmm6
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm6
+; X64-SSE2-NEXT: pand %xmm12, %xmm5
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm5
+; X64-SSE2-NEXT: packuswb %xmm6, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm14, %xmm6
+; X64-SSE2-NEXT: psrlw $1, %xmm6
+; X64-SSE2-NEXT: pand %xmm11, %xmm6
+; X64-SSE2-NEXT: psubb %xmm6, %xmm14
+; X64-SSE2-NEXT: movdqa %xmm14, %xmm6
+; X64-SSE2-NEXT: pand %xmm10, %xmm6
+; X64-SSE2-NEXT: psrlw $2, %xmm14
+; X64-SSE2-NEXT: pand %xmm10, %xmm14
+; X64-SSE2-NEXT: paddb %xmm6, %xmm14
+; X64-SSE2-NEXT: movdqa %xmm14, %xmm7
; X64-SSE2-NEXT: psrlw $4, %xmm7
-; X64-SSE2-NEXT: paddb %xmm6, %xmm7
-; X64-SSE2-NEXT: movdqa %xmm12, %xmm6
+; X64-SSE2-NEXT: paddb %xmm14, %xmm7
+; X64-SSE2-NEXT: movdqa %xmm13, %xmm6
; X64-SSE2-NEXT: psrlw $1, %xmm6
+; X64-SSE2-NEXT: pand %xmm11, %xmm6
+; X64-SSE2-NEXT: psubb %xmm6, %xmm13
+; X64-SSE2-NEXT: movdqa %xmm13, %xmm6
; X64-SSE2-NEXT: pand %xmm10, %xmm6
-; X64-SSE2-NEXT: psubb %xmm6, %xmm12
-; X64-SSE2-NEXT: movdqa %xmm12, %xmm6
-; X64-SSE2-NEXT: pand %xmm9, %xmm6
-; X64-SSE2-NEXT: psrlw $2, %xmm12
-; X64-SSE2-NEXT: pand %xmm9, %xmm12
-; X64-SSE2-NEXT: paddb %xmm6, %xmm12
-; X64-SSE2-NEXT: movdqa %xmm12, %xmm6
+; X64-SSE2-NEXT: psrlw $2, %xmm13
+; X64-SSE2-NEXT: pand %xmm10, %xmm13
+; X64-SSE2-NEXT: paddb %xmm6, %xmm13
+; X64-SSE2-NEXT: movdqa %xmm13, %xmm6
; X64-SSE2-NEXT: psrlw $4, %xmm6
-; X64-SSE2-NEXT: paddb %xmm12, %xmm6
-; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm12
-; X64-SSE2-NEXT: pand %xmm1, %xmm7
-; X64-SSE2-NEXT: pand %xmm1, %xmm6
-; X64-SSE2-NEXT: paddb %xmm7, %xmm6
-; X64-SSE2-NEXT: movdqa %xmm12, %xmm7
+; X64-SSE2-NEXT: paddb %xmm13, %xmm6
+; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm14
+; X64-SSE2-NEXT: pand %xmm12, %xmm7
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm7
+; X64-SSE2-NEXT: pand %xmm12, %xmm6
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm6
+; X64-SSE2-NEXT: packuswb %xmm7, %xmm6
+; X64-SSE2-NEXT: movdqa %xmm14, %xmm7
; X64-SSE2-NEXT: psrlw $1, %xmm7
+; X64-SSE2-NEXT: pand %xmm11, %xmm7
+; X64-SSE2-NEXT: psubb %xmm7, %xmm14
+; X64-SSE2-NEXT: movdqa %xmm14, %xmm7
; X64-SSE2-NEXT: pand %xmm10, %xmm7
-; X64-SSE2-NEXT: psubb %xmm7, %xmm12
-; X64-SSE2-NEXT: movdqa %xmm12, %xmm7
-; X64-SSE2-NEXT: pand %xmm9, %xmm7
-; X64-SSE2-NEXT: psrlw $2, %xmm12
-; X64-SSE2-NEXT: pand %xmm9, %xmm12
-; X64-SSE2-NEXT: paddb %xmm7, %xmm12
-; X64-SSE2-NEXT: movdqa %xmm12, %xmm13
+; X64-SSE2-NEXT: psrlw $2, %xmm14
+; X64-SSE2-NEXT: pand %xmm10, %xmm14
+; X64-SSE2-NEXT: paddb %xmm7, %xmm14
+; X64-SSE2-NEXT: movdqa %xmm14, %xmm13
; X64-SSE2-NEXT: psrlw $4, %xmm13
-; X64-SSE2-NEXT: paddb %xmm12, %xmm13
-; X64-SSE2-NEXT: movdqa %xmm11, %xmm7
+; X64-SSE2-NEXT: paddb %xmm14, %xmm13
+; X64-SSE2-NEXT: movdqa %xmm9, %xmm7
; X64-SSE2-NEXT: psrlw $1, %xmm7
+; X64-SSE2-NEXT: pand %xmm11, %xmm7
+; X64-SSE2-NEXT: psubb %xmm7, %xmm9
+; X64-SSE2-NEXT: movdqa %xmm9, %xmm7
; X64-SSE2-NEXT: pand %xmm10, %xmm7
-; X64-SSE2-NEXT: psubb %xmm7, %xmm11
-; X64-SSE2-NEXT: movdqa %xmm11, %xmm7
-; X64-SSE2-NEXT: pand %xmm9, %xmm7
-; X64-SSE2-NEXT: psrlw $2, %xmm11
-; X64-SSE2-NEXT: pand %xmm9, %xmm11
-; X64-SSE2-NEXT: paddb %xmm7, %xmm11
-; X64-SSE2-NEXT: movdqa %xmm11, %xmm7
+; X64-SSE2-NEXT: psrlw $2, %xmm9
+; X64-SSE2-NEXT: pand %xmm10, %xmm9
+; X64-SSE2-NEXT: paddb %xmm7, %xmm9
+; X64-SSE2-NEXT: movdqa %xmm9, %xmm7
; X64-SSE2-NEXT: psrlw $4, %xmm7
-; X64-SSE2-NEXT: paddb %xmm11, %xmm7
-; X64-SSE2-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm11
-; X64-SSE2-NEXT: pand %xmm1, %xmm13
-; X64-SSE2-NEXT: pand %xmm1, %xmm7
-; X64-SSE2-NEXT: paddb %xmm13, %xmm7
-; X64-SSE2-NEXT: movdqa %xmm11, %xmm12
-; X64-SSE2-NEXT: psrlw $1, %xmm12
-; X64-SSE2-NEXT: pand %xmm10, %xmm12
-; X64-SSE2-NEXT: psubb %xmm12, %xmm11
-; X64-SSE2-NEXT: movdqa %xmm11, %xmm12
-; X64-SSE2-NEXT: pand %xmm9, %xmm12
-; X64-SSE2-NEXT: psrlw $2, %xmm11
-; X64-SSE2-NEXT: pand %xmm9, %xmm11
-; X64-SSE2-NEXT: paddb %xmm12, %xmm11
-; X64-SSE2-NEXT: movdqa %xmm11, %xmm12
-; X64-SSE2-NEXT: psrlw $4, %xmm12
-; X64-SSE2-NEXT: paddb %xmm11, %xmm12
-; X64-SSE2-NEXT: movdqa %xmm8, %xmm11
-; X64-SSE2-NEXT: psrlw $1, %xmm11
-; X64-SSE2-NEXT: pand %xmm10, %xmm11
-; X64-SSE2-NEXT: psubb %xmm11, %xmm8
-; X64-SSE2-NEXT: movdqa %xmm8, %xmm10
-; X64-SSE2-NEXT: pand %xmm9, %xmm10
-; X64-SSE2-NEXT: psrlw $2, %xmm8
-; X64-SSE2-NEXT: pand %xmm9, %xmm8
-; X64-SSE2-NEXT: paddb %xmm10, %xmm8
-; X64-SSE2-NEXT: movdqa %xmm8, %xmm9
-; X64-SSE2-NEXT: psrlw $4, %xmm9
-; X64-SSE2-NEXT: paddb %xmm8, %xmm9
-; X64-SSE2-NEXT: pand %xmm1, %xmm12
-; X64-SSE2-NEXT: pand %xmm1, %xmm9
-; X64-SSE2-NEXT: paddb %xmm12, %xmm9
-; X64-SSE2-NEXT: pxor %xmm8, %xmm8
+; X64-SSE2-NEXT: paddb %xmm9, %xmm7
+; X64-SSE2-NEXT: pand %xmm12, %xmm13
+; X64-SSE2-NEXT: pand %xmm12, %xmm7
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm13
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm7
+; X64-SSE2-NEXT: packuswb %xmm13, %xmm7
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm0
; X64-SSE2-NEXT: psadbw %xmm8, %xmm0
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm1, %xmm0
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm3
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm3
; X64-SSE2-NEXT: psadbw %xmm8, %xmm3
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm3, %xmm1
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm2
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm2
; X64-SSE2-NEXT: psadbw %xmm8, %xmm2
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm2, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm4
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm4
; X64-SSE2-NEXT: psadbw %xmm8, %xmm4
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm4, %xmm2
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X64-SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; X64-SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm8, %xmm1
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm5
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm5
; X64-SSE2-NEXT: psadbw %xmm8, %xmm5
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm5[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm5, %xmm1
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm6
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm6
; X64-SSE2-NEXT: psadbw %xmm8, %xmm6
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm6[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm6, %xmm2
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm7
+; X64-SSE2-NEXT: packuswb %xmm8, %xmm7
; X64-SSE2-NEXT: psadbw %xmm8, %xmm7
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm7[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm7, %xmm2
-; X64-SSE2-NEXT: psadbw %xmm8, %xmm9
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm9[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm9, %xmm3
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm7[0],xmm6[1],xmm7[1]
+; X64-SSE2-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm6[0]
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: reduce_ctpop_v4i64_buildvector_v8i32:
@@ -4262,416 +4621,453 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; X86-SSE4-NEXT: movl %esp, %ebp
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $80, %esp
-; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm6
-; X86-SSE4-NEXT: movdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm5
-; X86-SSE4-NEXT: pand %xmm4, %xmm5
-; X86-SSE4-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm5, %xmm7
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm0
+; X86-SSE4-NEXT: movdqa {{.*#+}} xmm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm4
+; X86-SSE4-NEXT: pand %xmm7, %xmm4
+; X86-SSE4-NEXT: movdqa {{.*#+}} xmm6 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm5
+; X86-SSE4-NEXT: pshufb %xmm4, %xmm5
; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
+; X86-SSE4-NEXT: pand %xmm7, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm4
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm4
+; X86-SSE4-NEXT: paddb %xmm5, %xmm4
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm1
+; X86-SSE4-NEXT: pand %xmm7, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm5
; X86-SSE4-NEXT: pshufb %xmm1, %xmm5
-; X86-SSE4-NEXT: paddb %xmm7, %xmm5
+; X86-SSE4-NEXT: psrlw $4, %xmm3
+; X86-SSE4-NEXT: pand %xmm7, %xmm3
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm1
+; X86-SSE4-NEXT: pshufb %xmm3, %xmm1
+; X86-SSE4-NEXT: pxor %xmm3, %xmm3
+; X86-SSE4-NEXT: psadbw %xmm3, %xmm4
+; X86-SSE4-NEXT: paddb %xmm5, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm3, %xmm1
+; X86-SSE4-NEXT: packusdw %xmm4, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm7
+; X86-SSE4-NEXT: pand %xmm7, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm3
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm3
; X86-SSE4-NEXT: psrlw $4, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm1
+; X86-SSE4-NEXT: pand %xmm7, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm1
; X86-SSE4-NEXT: pshufb %xmm0, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm1
-; X86-SSE4-NEXT: paddb %xmm7, %xmm0
-; X86-SSE4-NEXT: paddb %xmm5, %xmm0
+; X86-SSE4-NEXT: paddb %xmm3, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE4-NEXT: pand %xmm7, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm4
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm4
+; X86-SSE4-NEXT: psrlw $4, %xmm2
+; X86-SSE4-NEXT: pand %xmm7, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm0
+; X86-SSE4-NEXT: pshufb %xmm2, %xmm0
+; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm2
+; X86-SSE4-NEXT: pxor %xmm3, %xmm3
+; X86-SSE4-NEXT: psadbw %xmm3, %xmm1
+; X86-SSE4-NEXT: paddb %xmm4, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm3, %xmm0
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm0
; X86-SSE4-NEXT: movdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm5
-; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm0
-; X86-SSE4-NEXT: paddb %xmm5, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE4-NEXT: pand %xmm7, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm1
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm1
; X86-SSE4-NEXT: psrlw $4, %xmm2
-; X86-SSE4-NEXT: pand %xmm4, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm7
-; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm1
-; X86-SSE4-NEXT: paddb %xmm5, %xmm7
-; X86-SSE4-NEXT: paddb %xmm0, %xmm7
-; X86-SSE4-NEXT: movdqa %xmm7, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
+; X86-SSE4-NEXT: pand %xmm7, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm0
+; X86-SSE4-NEXT: pshufb %xmm2, %xmm0
+; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm4
+; X86-SSE4-NEXT: paddb %xmm1, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm1
+; X86-SSE4-NEXT: pand %xmm7, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm5
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm5
+; X86-SSE4-NEXT: psrlw $4, %xmm4
+; X86-SSE4-NEXT: pand %xmm7, %xmm4
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm2
+; X86-SSE4-NEXT: pshufb %xmm4, %xmm2
+; X86-SSE4-NEXT: paddb %xmm5, %xmm2
+; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE4-NEXT: psadbw %xmm3, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm3, %xmm2
+; X86-SSE4-NEXT: packusdw %xmm0, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm2, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm2
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm2
+; X86-SSE4-NEXT: pand %xmm7, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm4
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm4
; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
+; X86-SSE4-NEXT: pand %xmm7, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm0
; X86-SSE4-NEXT: pshufb %xmm1, %xmm0
-; X86-SSE4-NEXT: paddb %xmm2, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm6, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm2
+; X86-SSE4-NEXT: paddb %xmm4, %xmm0
+; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm1
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm4
+; X86-SSE4-NEXT: pand %xmm7, %xmm4
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm5
+; X86-SSE4-NEXT: pshufb %xmm4, %xmm5
+; X86-SSE4-NEXT: psrlw $4, %xmm1
+; X86-SSE4-NEXT: pand %xmm7, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm2
; X86-SSE4-NEXT: pshufb %xmm1, %xmm2
-; X86-SSE4-NEXT: psrlw $4, %xmm6
-; X86-SSE4-NEXT: pand %xmm4, %xmm6
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm1
-; X86-SSE4-NEXT: pshufb %xmm6, %xmm1
-; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm5
-; X86-SSE4-NEXT: paddb %xmm2, %xmm1
-; X86-SSE4-NEXT: paddb %xmm0, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-SSE4-NEXT: movdqa %xmm5, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm2
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm2
-; X86-SSE4-NEXT: psrlw $4, %xmm5
-; X86-SSE4-NEXT: pand %xmm4, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
-; X86-SSE4-NEXT: pshufb %xmm5, %xmm0
-; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm5
-; X86-SSE4-NEXT: paddb %xmm2, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm5, %xmm2
-; X86-SSE4-NEXT: pand %xmm4, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm7
-; X86-SSE4-NEXT: psrlw $4, %xmm5
-; X86-SSE4-NEXT: pand %xmm4, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm1
-; X86-SSE4-NEXT: pshufb %xmm5, %xmm1
-; X86-SSE4-NEXT: paddb %xmm7, %xmm1
-; X86-SSE4-NEXT: movdqa 104(%ebp), %xmm2
-; X86-SSE4-NEXT: paddb %xmm0, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm1, (%esp) # 16-byte Spill
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm5
-; X86-SSE4-NEXT: psrlw $4, %xmm2
-; X86-SSE4-NEXT: pand %xmm4, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm0
-; X86-SSE4-NEXT: paddb %xmm5, %xmm0
-; X86-SSE4-NEXT: movdqa 88(%ebp), %xmm2
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm5
-; X86-SSE4-NEXT: pand %xmm4, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm1
-; X86-SSE4-NEXT: pshufb %xmm5, %xmm1
-; X86-SSE4-NEXT: psrlw $4, %xmm2
-; X86-SSE4-NEXT: pand %xmm4, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm7
-; X86-SSE4-NEXT: paddb %xmm1, %xmm7
-; X86-SSE4-NEXT: paddb %xmm0, %xmm7
-; X86-SSE4-NEXT: movdqa 136(%ebp), %xmm1
+; X86-SSE4-NEXT: paddb %xmm5, %xmm2
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm2
+; X86-SSE4-NEXT: packusdw %xmm0, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm2, (%esp) # 16-byte Spill
+; X86-SSE4-NEXT: movdqa 104(%ebp), %xmm1
; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm2
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm2
+; X86-SSE4-NEXT: pand %xmm7, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm4
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm4
; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
+; X86-SSE4-NEXT: pand %xmm7, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm0
; X86-SSE4-NEXT: pshufb %xmm1, %xmm0
-; X86-SSE4-NEXT: paddb %xmm2, %xmm0
-; X86-SSE4-NEXT: movdqa 120(%ebp), %xmm1
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm2
-; X86-SSE4-NEXT: pand %xmm4, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm5
+; X86-SSE4-NEXT: paddb %xmm4, %xmm0
+; X86-SSE4-NEXT: movdqa 88(%ebp), %xmm4
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm1
+; X86-SSE4-NEXT: pand %xmm7, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm7, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm5
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm5
+; X86-SSE4-NEXT: psrlw $4, %xmm4
+; X86-SSE4-NEXT: pand %xmm7, %xmm4
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm4, %xmm7
+; X86-SSE4-NEXT: paddb %xmm5, %xmm7
+; X86-SSE4-NEXT: psadbw %xmm3, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm3, %xmm7
+; X86-SSE4-NEXT: packusdw %xmm0, %xmm7
+; X86-SSE4-NEXT: movdqa 136(%ebp), %xmm0
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm4
+; X86-SSE4-NEXT: pand %xmm2, %xmm4
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm5
+; X86-SSE4-NEXT: pshufb %xmm4, %xmm5
+; X86-SSE4-NEXT: psrlw $4, %xmm0
+; X86-SSE4-NEXT: pand %xmm2, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm4
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm4
+; X86-SSE4-NEXT: paddb %xmm5, %xmm4
+; X86-SSE4-NEXT: movdqa 120(%ebp), %xmm5
+; X86-SSE4-NEXT: movdqa %xmm5, %xmm0
+; X86-SSE4-NEXT: pand %xmm2, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm1
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm1
+; X86-SSE4-NEXT: psrlw $4, %xmm5
+; X86-SSE4-NEXT: pand %xmm2, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm3
+; X86-SSE4-NEXT: pshufb %xmm5, %xmm3
+; X86-SSE4-NEXT: paddb %xmm1, %xmm3
+; X86-SSE4-NEXT: pxor %xmm0, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm4
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm3
+; X86-SSE4-NEXT: packusdw %xmm4, %xmm3
+; X86-SSE4-NEXT: movdqa 168(%ebp), %xmm1
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm4
+; X86-SSE4-NEXT: pand %xmm2, %xmm4
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm0
+; X86-SSE4-NEXT: pshufb %xmm4, %xmm0
; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm6
-; X86-SSE4-NEXT: paddb %xmm5, %xmm6
-; X86-SSE4-NEXT: paddb %xmm0, %xmm6
-; X86-SSE4-NEXT: movdqa 168(%ebp), %xmm0
+; X86-SSE4-NEXT: movdqa %xmm2, %xmm4
+; X86-SSE4-NEXT: pand %xmm2, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm5
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm5
+; X86-SSE4-NEXT: paddb %xmm0, %xmm5
+; X86-SSE4-NEXT: movdqa 152(%ebp), %xmm0
; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm2
+; X86-SSE4-NEXT: pand %xmm2, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm2
; X86-SSE4-NEXT: pshufb %xmm1, %xmm2
; X86-SSE4-NEXT: psrlw $4, %xmm0
; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm5
-; X86-SSE4-NEXT: paddb %xmm2, %xmm5
-; X86-SSE4-NEXT: movdqa 152(%ebp), %xmm1
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm2
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm2
-; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm4
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm4
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm4
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm4
; X86-SSE4-NEXT: paddb %xmm2, %xmm4
-; X86-SSE4-NEXT: paddb %xmm5, %xmm4
-; X86-SSE4-NEXT: movdqa 200(%ebp), %xmm1
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm0, %xmm5
-; X86-SSE4-NEXT: pand %xmm0, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm0
-; X86-SSE4-NEXT: psrlw $4, %xmm1
+; X86-SSE4-NEXT: pxor %xmm0, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm5
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm4
+; X86-SSE4-NEXT: packusdw %xmm5, %xmm4
+; X86-SSE4-NEXT: movdqa 200(%ebp), %xmm0
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE4-NEXT: movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
; X86-SSE4-NEXT: pand %xmm5, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm5, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm5
-; X86-SSE4-NEXT: paddb %xmm0, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm2
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm2
+; X86-SSE4-NEXT: psrlw $4, %xmm0
+; X86-SSE4-NEXT: pand %xmm5, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm5
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm5
+; X86-SSE4-NEXT: paddb %xmm2, %xmm5
; X86-SSE4-NEXT: movdqa 184(%ebp), %xmm0
; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
-; X86-SSE4-NEXT: pand %xmm2, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm2
+; X86-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm2
; X86-SSE4-NEXT: pshufb %xmm1, %xmm2
; X86-SSE4-NEXT: psrlw $4, %xmm0
; X86-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm3
-; X86-SSE4-NEXT: paddb %xmm2, %xmm3
-; X86-SSE4-NEXT: paddb %xmm5, %xmm3
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm6
+; X86-SSE4-NEXT: paddb %xmm2, %xmm6
; X86-SSE4-NEXT: pxor %xmm0, %xmm0
-; X86-SSE4-NEXT: movdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm1, %xmm5
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm5
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm6
+; X86-SSE4-NEXT: packusdw %xmm5, %xmm6
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: movdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm0
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm0
; X86-SSE4-NEXT: movdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm2
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm2, %xmm1
-; X86-SSE4-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm2
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm2
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm2
+; X86-SSE4-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; X86-SSE4-NEXT: movdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm2
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm2, %xmm1
-; X86-SSE4-NEXT: movdqa (%esp), %xmm0 # 16-byte Reload
-; X86-SSE4-NEXT: pxor %xmm2, %xmm2
-; X86-SSE4-NEXT: psadbw %xmm2, %xmm0
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm0, %xmm2
-; X86-SSE4-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X86-SSE4-NEXT: punpcklqdq {{.*#+}} xmm5 = xmm5[0],xmm1[0]
-; X86-SSE4-NEXT: pxor %xmm0, %xmm0
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm7
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm7, %xmm1
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm6
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm6[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm6, %xmm2
-; X86-SSE4-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm4
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm4, %xmm2
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm3
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm3, %xmm4
-; X86-SSE4-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
-; X86-SSE4-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
-; X86-SSE4-NEXT: movdqa %xmm5, %xmm0
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm2
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm2
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm2
+; X86-SSE4-NEXT: movdqa (%esp), %xmm5 # 16-byte Reload
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm5
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm5
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm5
+; X86-SSE4-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; X86-SSE4-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm7
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm7
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm7
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm3
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm3
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm3
+; X86-SSE4-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm3[0],xmm7[1],xmm3[1]
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm4
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm4
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm4
+; X86-SSE4-NEXT: packusdw %xmm1, %xmm6
+; X86-SSE4-NEXT: packuswb %xmm1, %xmm6
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm6
+; X86-SSE4-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm6[0],xmm4[1],xmm6[1]
+; X86-SSE4-NEXT: punpcklqdq {{.*#+}} xmm7 = xmm7[0],xmm4[0]
+; X86-SSE4-NEXT: movdqa %xmm7, %xmm1
; X86-SSE4-NEXT: movl %ebp, %esp
; X86-SSE4-NEXT: popl %ebp
; X86-SSE4-NEXT: retl
;
; X64-SSE4-LABEL: reduce_ctpop_v4i64_buildvector_v8i32:
; X64-SSE4: # %bb.0:
-; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm9
-; X64-SSE4-NEXT: movdqa {{.*#+}} xmm10 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-SSE4-NEXT: movdqa %xmm1, %xmm11
-; X64-SSE4-NEXT: pand %xmm10, %xmm11
-; X64-SSE4-NEXT: movdqa {{.*#+}} xmm8 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm12
-; X64-SSE4-NEXT: pshufb %xmm11, %xmm12
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm8
+; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm10
+; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm12
+; X64-SSE4-NEXT: movdqa {{.*#+}} xmm11 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE4-NEXT: pand %xmm11, %xmm0
+; X64-SSE4-NEXT: movdqa {{.*#+}} xmm9 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm13
+; X64-SSE4-NEXT: pshufb %xmm0, %xmm13
; X64-SSE4-NEXT: psrlw $4, %xmm1
-; X64-SSE4-NEXT: pand %xmm10, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
-; X64-SSE4-NEXT: pshufb %xmm1, %xmm11
-; X64-SSE4-NEXT: paddb %xmm12, %xmm11
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm1
-; X64-SSE4-NEXT: pand %xmm10, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm12
-; X64-SSE4-NEXT: pshufb %xmm1, %xmm12
-; X64-SSE4-NEXT: psrlw $4, %xmm0
-; X64-SSE4-NEXT: pand %xmm10, %xmm0
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm1
+; X64-SSE4-NEXT: pand %xmm11, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm14
+; X64-SSE4-NEXT: pshufb %xmm1, %xmm14
+; X64-SSE4-NEXT: paddb %xmm13, %xmm14
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm0
+; X64-SSE4-NEXT: pand %xmm11, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm1
; X64-SSE4-NEXT: pshufb %xmm0, %xmm1
-; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
-; X64-SSE4-NEXT: paddb %xmm12, %xmm1
-; X64-SSE4-NEXT: paddb %xmm11, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm3, %xmm11
-; X64-SSE4-NEXT: pand %xmm10, %xmm11
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm12
-; X64-SSE4-NEXT: pshufb %xmm11, %xmm12
+; X64-SSE4-NEXT: psrlw $4, %xmm8
+; X64-SSE4-NEXT: pand %xmm11, %xmm8
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm0
+; X64-SSE4-NEXT: pshufb %xmm8, %xmm0
+; X64-SSE4-NEXT: pxor %xmm8, %xmm8
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm14
+; X64-SSE4-NEXT: paddb %xmm1, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm14, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm3, %xmm1
+; X64-SSE4-NEXT: pand %xmm11, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm13
+; X64-SSE4-NEXT: pshufb %xmm1, %xmm13
; X64-SSE4-NEXT: psrlw $4, %xmm3
-; X64-SSE4-NEXT: pand %xmm10, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm13
-; X64-SSE4-NEXT: pshufb %xmm3, %xmm13
-; X64-SSE4-NEXT: paddb %xmm12, %xmm13
+; X64-SSE4-NEXT: pand %xmm11, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm1
+; X64-SSE4-NEXT: pshufb %xmm3, %xmm1
+; X64-SSE4-NEXT: paddb %xmm13, %xmm1
; X64-SSE4-NEXT: movdqa %xmm2, %xmm3
-; X64-SSE4-NEXT: pand %xmm10, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm12
-; X64-SSE4-NEXT: pshufb %xmm3, %xmm12
+; X64-SSE4-NEXT: pand %xmm11, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm14
+; X64-SSE4-NEXT: pshufb %xmm3, %xmm14
; X64-SSE4-NEXT: psrlw $4, %xmm2
-; X64-SSE4-NEXT: pand %xmm10, %xmm2
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm3
+; X64-SSE4-NEXT: pand %xmm11, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm3
; X64-SSE4-NEXT: pshufb %xmm2, %xmm3
-; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm11
-; X64-SSE4-NEXT: paddb %xmm12, %xmm3
-; X64-SSE4-NEXT: paddb %xmm13, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm5, %xmm2
-; X64-SSE4-NEXT: pand %xmm10, %xmm2
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm12
-; X64-SSE4-NEXT: pshufb %xmm2, %xmm12
+; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm13
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm1
+; X64-SSE4-NEXT: paddb %xmm14, %xmm3
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm3
+; X64-SSE4-NEXT: packusdw %xmm1, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm5, %xmm1
+; X64-SSE4-NEXT: pand %xmm11, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm2
+; X64-SSE4-NEXT: pshufb %xmm1, %xmm2
; X64-SSE4-NEXT: psrlw $4, %xmm5
-; X64-SSE4-NEXT: pand %xmm10, %xmm5
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm13
-; X64-SSE4-NEXT: pshufb %xmm5, %xmm13
-; X64-SSE4-NEXT: paddb %xmm12, %xmm13
+; X64-SSE4-NEXT: pand %xmm11, %xmm5
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm1
+; X64-SSE4-NEXT: pshufb %xmm5, %xmm1
+; X64-SSE4-NEXT: paddb %xmm2, %xmm1
; X64-SSE4-NEXT: movdqa %xmm4, %xmm2
-; X64-SSE4-NEXT: pand %xmm10, %xmm2
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm5
-; X64-SSE4-NEXT: pshufb %xmm2, %xmm5
+; X64-SSE4-NEXT: pand %xmm11, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm14
+; X64-SSE4-NEXT: pshufb %xmm2, %xmm14
; X64-SSE4-NEXT: psrlw $4, %xmm4
-; X64-SSE4-NEXT: pand %xmm10, %xmm4
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm2
+; X64-SSE4-NEXT: pand %xmm11, %xmm4
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm2
; X64-SSE4-NEXT: pshufb %xmm4, %xmm2
-; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm12
-; X64-SSE4-NEXT: paddb %xmm5, %xmm2
-; X64-SSE4-NEXT: paddb %xmm13, %xmm2
-; X64-SSE4-NEXT: movdqa %xmm7, %xmm4
-; X64-SSE4-NEXT: pand %xmm10, %xmm4
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm5
-; X64-SSE4-NEXT: pshufb %xmm4, %xmm5
+; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm5
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm1
+; X64-SSE4-NEXT: paddb %xmm14, %xmm2
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm2
+; X64-SSE4-NEXT: packusdw %xmm1, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm7, %xmm1
+; X64-SSE4-NEXT: pand %xmm11, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm4
+; X64-SSE4-NEXT: pshufb %xmm1, %xmm4
; X64-SSE4-NEXT: psrlw $4, %xmm7
-; X64-SSE4-NEXT: pand %xmm10, %xmm7
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm13
-; X64-SSE4-NEXT: pshufb %xmm7, %xmm13
-; X64-SSE4-NEXT: paddb %xmm5, %xmm13
+; X64-SSE4-NEXT: pand %xmm11, %xmm7
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm1
+; X64-SSE4-NEXT: pshufb %xmm7, %xmm1
+; X64-SSE4-NEXT: paddb %xmm4, %xmm1
; X64-SSE4-NEXT: movdqa %xmm6, %xmm4
-; X64-SSE4-NEXT: pand %xmm10, %xmm4
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm5
-; X64-SSE4-NEXT: pshufb %xmm4, %xmm5
+; X64-SSE4-NEXT: pand %xmm11, %xmm4
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm7
+; X64-SSE4-NEXT: pshufb %xmm4, %xmm7
; X64-SSE4-NEXT: psrlw $4, %xmm6
-; X64-SSE4-NEXT: pand %xmm10, %xmm6
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm4
+; X64-SSE4-NEXT: pand %xmm11, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm4
; X64-SSE4-NEXT: pshufb %xmm6, %xmm4
; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm6
-; X64-SSE4-NEXT: paddb %xmm5, %xmm4
-; X64-SSE4-NEXT: paddb %xmm13, %xmm4
-; X64-SSE4-NEXT: movdqa %xmm6, %xmm5
-; X64-SSE4-NEXT: pand %xmm10, %xmm5
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm7
-; X64-SSE4-NEXT: pshufb %xmm5, %xmm7
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm1
+; X64-SSE4-NEXT: paddb %xmm7, %xmm4
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm4
+; X64-SSE4-NEXT: packusdw %xmm1, %xmm4
+; X64-SSE4-NEXT: movdqa %xmm6, %xmm1
+; X64-SSE4-NEXT: pand %xmm11, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm7
+; X64-SSE4-NEXT: pshufb %xmm1, %xmm7
; X64-SSE4-NEXT: psrlw $4, %xmm6
-; X64-SSE4-NEXT: pand %xmm10, %xmm6
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm13
-; X64-SSE4-NEXT: pshufb %xmm6, %xmm13
-; X64-SSE4-NEXT: paddb %xmm7, %xmm13
-; X64-SSE4-NEXT: movdqa %xmm12, %xmm5
-; X64-SSE4-NEXT: pand %xmm10, %xmm5
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm6
+; X64-SSE4-NEXT: pand %xmm11, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm14
+; X64-SSE4-NEXT: pshufb %xmm6, %xmm14
+; X64-SSE4-NEXT: paddb %xmm7, %xmm14
+; X64-SSE4-NEXT: movdqa %xmm5, %xmm1
+; X64-SSE4-NEXT: pand %xmm11, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm6
+; X64-SSE4-NEXT: pshufb %xmm1, %xmm6
+; X64-SSE4-NEXT: psrlw $4, %xmm5
+; X64-SSE4-NEXT: pand %xmm11, %xmm5
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm1
+; X64-SSE4-NEXT: pshufb %xmm5, %xmm1
+; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm5
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm14
+; X64-SSE4-NEXT: paddb %xmm6, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm1
+; X64-SSE4-NEXT: packusdw %xmm14, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm5, %xmm6
+; X64-SSE4-NEXT: pand %xmm11, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm7
+; X64-SSE4-NEXT: pshufb %xmm6, %xmm7
+; X64-SSE4-NEXT: psrlw $4, %xmm5
+; X64-SSE4-NEXT: pand %xmm11, %xmm5
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm6
; X64-SSE4-NEXT: pshufb %xmm5, %xmm6
-; X64-SSE4-NEXT: psrlw $4, %xmm12
-; X64-SSE4-NEXT: pand %xmm10, %xmm12
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm5
-; X64-SSE4-NEXT: pshufb %xmm12, %xmm5
-; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm7
-; X64-SSE4-NEXT: paddb %xmm6, %xmm5
-; X64-SSE4-NEXT: paddb %xmm13, %xmm5
-; X64-SSE4-NEXT: movdqa %xmm7, %xmm6
-; X64-SSE4-NEXT: pand %xmm10, %xmm6
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm12
-; X64-SSE4-NEXT: pshufb %xmm6, %xmm12
-; X64-SSE4-NEXT: psrlw $4, %xmm7
-; X64-SSE4-NEXT: pand %xmm10, %xmm7
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm13
-; X64-SSE4-NEXT: pshufb %xmm7, %xmm13
-; X64-SSE4-NEXT: paddb %xmm12, %xmm13
-; X64-SSE4-NEXT: movdqa %xmm11, %xmm6
-; X64-SSE4-NEXT: pand %xmm10, %xmm6
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm7
+; X64-SSE4-NEXT: paddb %xmm7, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm13, %xmm5
+; X64-SSE4-NEXT: pand %xmm11, %xmm5
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm7
+; X64-SSE4-NEXT: pshufb %xmm5, %xmm7
+; X64-SSE4-NEXT: psrlw $4, %xmm13
+; X64-SSE4-NEXT: pand %xmm11, %xmm13
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm5
+; X64-SSE4-NEXT: pshufb %xmm13, %xmm5
+; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm13
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm6
+; X64-SSE4-NEXT: paddb %xmm7, %xmm5
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm5
+; X64-SSE4-NEXT: packusdw %xmm6, %xmm5
+; X64-SSE4-NEXT: movdqa %xmm13, %xmm6
+; X64-SSE4-NEXT: pand %xmm11, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm7
+; X64-SSE4-NEXT: pshufb %xmm6, %xmm7
+; X64-SSE4-NEXT: psrlw $4, %xmm13
+; X64-SSE4-NEXT: pand %xmm11, %xmm13
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm14
+; X64-SSE4-NEXT: pshufb %xmm13, %xmm14
+; X64-SSE4-NEXT: paddb %xmm7, %xmm14
+; X64-SSE4-NEXT: movdqa %xmm12, %xmm6
+; X64-SSE4-NEXT: pand %xmm11, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm7
; X64-SSE4-NEXT: pshufb %xmm6, %xmm7
-; X64-SSE4-NEXT: psrlw $4, %xmm11
-; X64-SSE4-NEXT: pand %xmm10, %xmm11
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm6
-; X64-SSE4-NEXT: pshufb %xmm11, %xmm6
-; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm11
+; X64-SSE4-NEXT: psrlw $4, %xmm12
+; X64-SSE4-NEXT: pand %xmm11, %xmm12
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm6
+; X64-SSE4-NEXT: pshufb %xmm12, %xmm6
+; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm12
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm14
; X64-SSE4-NEXT: paddb %xmm7, %xmm6
-; X64-SSE4-NEXT: paddb %xmm13, %xmm6
-; X64-SSE4-NEXT: movdqa %xmm11, %xmm7
-; X64-SSE4-NEXT: pand %xmm10, %xmm7
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm12
-; X64-SSE4-NEXT: pshufb %xmm7, %xmm12
-; X64-SSE4-NEXT: psrlw $4, %xmm11
-; X64-SSE4-NEXT: pand %xmm10, %xmm11
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm13
-; X64-SSE4-NEXT: pshufb %xmm11, %xmm13
-; X64-SSE4-NEXT: paddb %xmm12, %xmm13
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm7
-; X64-SSE4-NEXT: pand %xmm10, %xmm7
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
-; X64-SSE4-NEXT: pshufb %xmm7, %xmm11
-; X64-SSE4-NEXT: psrlw $4, %xmm0
-; X64-SSE4-NEXT: pand %xmm10, %xmm0
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm7
-; X64-SSE4-NEXT: pshufb %xmm0, %xmm7
-; X64-SSE4-NEXT: movdqa {{[0-9]+}}(%rsp), %xmm0
-; X64-SSE4-NEXT: paddb %xmm11, %xmm7
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm6
+; X64-SSE4-NEXT: packusdw %xmm14, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm12, %xmm7
+; X64-SSE4-NEXT: pand %xmm11, %xmm7
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm13
+; X64-SSE4-NEXT: pshufb %xmm7, %xmm13
+; X64-SSE4-NEXT: psrlw $4, %xmm12
+; X64-SSE4-NEXT: pand %xmm11, %xmm12
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm7
+; X64-SSE4-NEXT: pshufb %xmm12, %xmm7
; X64-SSE4-NEXT: paddb %xmm13, %xmm7
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm11
-; X64-SSE4-NEXT: pand %xmm10, %xmm11
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm12
-; X64-SSE4-NEXT: pshufb %xmm11, %xmm12
-; X64-SSE4-NEXT: psrlw $4, %xmm0
-; X64-SSE4-NEXT: pand %xmm10, %xmm0
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
-; X64-SSE4-NEXT: pshufb %xmm0, %xmm11
-; X64-SSE4-NEXT: paddb %xmm12, %xmm11
-; X64-SSE4-NEXT: movdqa %xmm9, %xmm0
-; X64-SSE4-NEXT: pand %xmm10, %xmm0
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm12
-; X64-SSE4-NEXT: pshufb %xmm0, %xmm12
-; X64-SSE4-NEXT: psrlw $4, %xmm9
-; X64-SSE4-NEXT: pand %xmm10, %xmm9
-; X64-SSE4-NEXT: pshufb %xmm9, %xmm8
-; X64-SSE4-NEXT: paddb %xmm12, %xmm8
-; X64-SSE4-NEXT: paddb %xmm11, %xmm8
-; X64-SSE4-NEXT: pxor %xmm9, %xmm9
-; X64-SSE4-NEXT: psadbw %xmm9, %xmm1
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm1, %xmm0
-; X64-SSE4-NEXT: psadbw %xmm9, %xmm3
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm3, %xmm1
-; X64-SSE4-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-SSE4-NEXT: psadbw %xmm9, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm2, %xmm1
-; X64-SSE4-NEXT: psadbw %xmm9, %xmm4
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm4, %xmm2
-; X64-SSE4-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X64-SSE4-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X64-SSE4-NEXT: psadbw %xmm9, %xmm5
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm5[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm5, %xmm1
-; X64-SSE4-NEXT: psadbw %xmm9, %xmm6
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm6[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm6, %xmm2
-; X64-SSE4-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X64-SSE4-NEXT: psadbw %xmm9, %xmm7
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm7[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm7, %xmm2
-; X64-SSE4-NEXT: psadbw %xmm9, %xmm8
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm8[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm8, %xmm3
-; X64-SSE4-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-SSE4-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X64-SSE4-NEXT: movdqa %xmm10, %xmm12
+; X64-SSE4-NEXT: pand %xmm11, %xmm12
+; X64-SSE4-NEXT: movdqa %xmm9, %xmm13
+; X64-SSE4-NEXT: pshufb %xmm12, %xmm13
+; X64-SSE4-NEXT: psrlw $4, %xmm10
+; X64-SSE4-NEXT: pand %xmm11, %xmm10
+; X64-SSE4-NEXT: pshufb %xmm10, %xmm9
+; X64-SSE4-NEXT: paddb %xmm13, %xmm9
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm7
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm9
+; X64-SSE4-NEXT: packusdw %xmm7, %xmm9
+; X64-SSE4-NEXT: packusdw %xmm8, %xmm0
+; X64-SSE4-NEXT: packuswb %xmm8, %xmm0
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm0
+; X64-SSE4-NEXT: packusdw %xmm8, %xmm3
+; X64-SSE4-NEXT: packuswb %xmm8, %xmm3
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm3
+; X64-SSE4-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; X64-SSE4-NEXT: packusdw %xmm8, %xmm2
+; X64-SSE4-NEXT: packuswb %xmm8, %xmm2
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm2
+; X64-SSE4-NEXT: packusdw %xmm8, %xmm4
+; X64-SSE4-NEXT: packuswb %xmm8, %xmm4
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm4
+; X64-SSE4-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1]
+; X64-SSE4-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; X64-SSE4-NEXT: packusdw %xmm8, %xmm1
+; X64-SSE4-NEXT: packuswb %xmm8, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm1
+; X64-SSE4-NEXT: packusdw %xmm8, %xmm5
+; X64-SSE4-NEXT: packuswb %xmm8, %xmm5
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm5
+; X64-SSE4-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
+; X64-SSE4-NEXT: packusdw %xmm8, %xmm6
+; X64-SSE4-NEXT: packuswb %xmm8, %xmm6
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm6
+; X64-SSE4-NEXT: packusdw %xmm8, %xmm9
+; X64-SSE4-NEXT: packuswb %xmm8, %xmm9
+; X64-SSE4-NEXT: psadbw %xmm8, %xmm9
+; X64-SSE4-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm9[0],xmm6[1],xmm9[1]
+; X64-SSE4-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm6[0]
; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: reduce_ctpop_v4i64_buildvector_v8i32:
@@ -4680,321 +5076,350 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; X86-AVX1-NEXT: movl %esp, %ebp
; X86-AVX1-NEXT: andl $-32, %esp
; X86-AVX1-NEXT: subl $96, %esp
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5
-; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm6
-; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
-; X86-AVX1-NEXT: vpsrlw $4, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm6
-; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX1-NEXT: vpand %xmm4, %xmm3, %xmm6
+; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpshufb %xmm3, %xmm5, %xmm3
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm3, %xmm6
+; X86-AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm6, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
+; X86-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
; X86-AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; X86-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
; X86-AVX1-NEXT: vpaddb %xmm1, %xmm2, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
; X86-AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-AVX1-NEXT: vmovdqa 8(%ebp), %xmm0
-; X86-AVX1-NEXT: vmovdqa 24(%ebp), %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; X86-AVX1-NEXT: vmovdqa 24(%ebp), %xmm0
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovdqa 8(%ebp), %xmm1
+; X86-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
; X86-AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-AVX1-NEXT: vmovdqa 56(%ebp), %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovdqa 40(%ebp), %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm6
-; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm0, %xmm1, %xmm0
+; X86-AVX1-NEXT: vmovdqa %xmm0, (%esp) # 16-byte Spill
; X86-AVX1-NEXT: vmovdqa 88(%ebp), %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovdqa 72(%ebp), %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm7
-; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
-; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm7, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm7
-; X86-AVX1-NEXT: vmovdqa 120(%ebp), %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vmovdqa 104(%ebp), %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm2
-; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; X86-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm2
+; X86-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm5, %xmm1
; X86-AVX1-NEXT: vpaddb %xmm2, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm5
-; X86-AVX1-NEXT: vmovdqa 152(%ebp), %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm2
-; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm0, %xmm1, %xmm7
+; X86-AVX1-NEXT: vmovdqa 120(%ebp), %xmm0
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm2
+; X86-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm1
-; X86-AVX1-NEXT: vmovdqa 136(%ebp), %xmm2
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm0
+; X86-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovdqa 104(%ebp), %xmm2
+; X86-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vpaddb %xmm0, %xmm2, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm3
-; X86-AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpsadbw %xmm0, %xmm6, %xmm1
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw %xmm0, %xmm7, %xmm2
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X86-AVX1-NEXT: vpsadbw %xmm0, %xmm5, %xmm1
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
-; X86-AVX1-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm5 # 16-byte Folded Reload
-; X86-AVX1-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm6 # 16-byte Folded Reload
-; X86-AVX1-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm7 # 16-byte Folded Reload
-; X86-AVX1-NEXT: vpsadbw %xmm0, %xmm3, %xmm0
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm7, %xmm1
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm6[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm6, %xmm3
-; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm5, %xmm3
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm5, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X86-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; X86-AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm2
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; X86-AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X86-AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; X86-AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; X86-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpackusdw %xmm0, %xmm2, %xmm1
+; X86-AVX1-NEXT: vmovdqa 152(%ebp), %xmm2
+; X86-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm2, %xmm2
+; X86-AVX1-NEXT: vmovdqa 136(%ebp), %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm0
+; X86-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm4
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm4, %xmm5, %xmm4
+; X86-AVX1-NEXT: vpaddb %xmm0, %xmm4, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm2, %xmm0, %xmm4
+; X86-AVX1-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X86-AVX1-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm2 # 16-byte Reload
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; X86-AVX1-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm5 # 16-byte Reload
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; X86-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm0[0],xmm2[0]
+; X86-AVX1-NEXT: vmovdqa (%esp), %xmm0 # 16-byte Reload
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm7, %xmm5
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpackusdw %xmm3, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpackuswb %xmm3, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpsadbw %xmm3, %xmm4, %xmm3
+; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; X86-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X86-AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm2, %ymm0
; X86-AVX1-NEXT: movl %ebp, %esp
; X86-AVX1-NEXT: popl %ebp
; X86-AVX1-NEXT: retl
;
; X64-AVX1-LABEL: reduce_ctpop_v4i64_buildvector_v8i32:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm10
-; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm8 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm11
-; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm9 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
-; X64-AVX1-NEXT: vpsrlw $4, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpshufb %xmm10, %xmm9, %xmm10
-; X64-AVX1-NEXT: vpaddb %xmm11, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm0, %xmm8, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm8
+; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm9 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX1-NEXT: vpand %xmm9, %xmm8, %xmm11
+; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm10 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX1-NEXT: vpshufb %xmm11, %xmm10, %xmm11
+; X64-AVX1-NEXT: vpsrlw $4, %xmm8, %xmm8
+; X64-AVX1-NEXT: vpand %xmm9, %xmm8, %xmm8
+; X64-AVX1-NEXT: vpshufb %xmm8, %xmm10, %xmm8
+; X64-AVX1-NEXT: vpaddb %xmm11, %xmm8, %xmm11
+; X64-AVX1-NEXT: vpxor %xmm8, %xmm8, %xmm8
+; X64-AVX1-NEXT: vpsadbw %xmm8, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm0, %xmm9, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpand %xmm0, %xmm8, %xmm0
-; X64-AVX1-NEXT: vpshufb %xmm0, %xmm9, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm0, %xmm11, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm0, %xmm10, %xmm0
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
-; X64-AVX1-NEXT: vpsrlw $4, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpshufb %xmm10, %xmm9, %xmm10
-; X64-AVX1-NEXT: vpaddb %xmm11, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm1, %xmm8, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
+; X64-AVX1-NEXT: vpand %xmm0, %xmm9, %xmm0
+; X64-AVX1-NEXT: vpshufb %xmm0, %xmm10, %xmm0
+; X64-AVX1-NEXT: vpaddb %xmm0, %xmm12, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm0, %xmm8, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm11, %xmm0, %xmm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
+; X64-AVX1-NEXT: vpsrlw $4, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpshufb %xmm11, %xmm10, %xmm11
+; X64-AVX1-NEXT: vpaddb %xmm12, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpsadbw %xmm8, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm1, %xmm9, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpand %xmm1, %xmm8, %xmm1
-; X64-AVX1-NEXT: vpshufb %xmm1, %xmm9, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm1, %xmm11, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm1, %xmm10, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
-; X64-AVX1-NEXT: vpsrlw $4, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpshufb %xmm10, %xmm9, %xmm10
-; X64-AVX1-NEXT: vpaddb %xmm11, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm2, %xmm8, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
+; X64-AVX1-NEXT: vpand %xmm1, %xmm9, %xmm1
+; X64-AVX1-NEXT: vpshufb %xmm1, %xmm10, %xmm1
+; X64-AVX1-NEXT: vpaddb %xmm1, %xmm12, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm8, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm11, %xmm1, %xmm1
+; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
+; X64-AVX1-NEXT: vpsrlw $4, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpshufb %xmm11, %xmm10, %xmm11
+; X64-AVX1-NEXT: vpaddb %xmm12, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpsadbw %xmm8, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm2, %xmm9, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
; X64-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpand %xmm2, %xmm8, %xmm2
-; X64-AVX1-NEXT: vpshufb %xmm2, %xmm9, %xmm2
-; X64-AVX1-NEXT: vpaddb %xmm2, %xmm11, %xmm2
-; X64-AVX1-NEXT: vpaddb %xmm2, %xmm10, %xmm2
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
-; X64-AVX1-NEXT: vpsrlw $4, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpshufb %xmm10, %xmm9, %xmm10
-; X64-AVX1-NEXT: vpaddb %xmm11, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm3, %xmm8, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
+; X64-AVX1-NEXT: vpand %xmm2, %xmm9, %xmm2
+; X64-AVX1-NEXT: vpshufb %xmm2, %xmm10, %xmm2
+; X64-AVX1-NEXT: vpaddb %xmm2, %xmm12, %xmm2
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm8, %xmm2
+; X64-AVX1-NEXT: vpackusdw %xmm11, %xmm2, %xmm2
+; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
+; X64-AVX1-NEXT: vpsrlw $4, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpshufb %xmm11, %xmm10, %xmm11
+; X64-AVX1-NEXT: vpaddb %xmm12, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpsadbw %xmm8, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm3, %xmm9, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
; X64-AVX1-NEXT: vpsrlw $4, %xmm3, %xmm3
-; X64-AVX1-NEXT: vpand %xmm3, %xmm8, %xmm3
-; X64-AVX1-NEXT: vpshufb %xmm3, %xmm9, %xmm3
-; X64-AVX1-NEXT: vpaddb %xmm3, %xmm11, %xmm3
-; X64-AVX1-NEXT: vpaddb %xmm3, %xmm10, %xmm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm4, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
-; X64-AVX1-NEXT: vpsrlw $4, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpshufb %xmm10, %xmm9, %xmm10
-; X64-AVX1-NEXT: vpaddb %xmm11, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm4, %xmm8, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
+; X64-AVX1-NEXT: vpand %xmm3, %xmm9, %xmm3
+; X64-AVX1-NEXT: vpshufb %xmm3, %xmm10, %xmm3
+; X64-AVX1-NEXT: vpaddb %xmm3, %xmm12, %xmm3
+; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm8, %xmm3
+; X64-AVX1-NEXT: vpackusdw %xmm11, %xmm3, %xmm3
+; X64-AVX1-NEXT: vextractf128 $1, %ymm4, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
+; X64-AVX1-NEXT: vpsrlw $4, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpshufb %xmm11, %xmm10, %xmm11
+; X64-AVX1-NEXT: vpaddb %xmm12, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpsadbw %xmm8, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm4, %xmm9, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
; X64-AVX1-NEXT: vpsrlw $4, %xmm4, %xmm4
-; X64-AVX1-NEXT: vpand %xmm4, %xmm8, %xmm4
-; X64-AVX1-NEXT: vpshufb %xmm4, %xmm9, %xmm4
-; X64-AVX1-NEXT: vpaddb %xmm4, %xmm11, %xmm4
-; X64-AVX1-NEXT: vpaddb %xmm4, %xmm10, %xmm4
-; X64-AVX1-NEXT: vextractf128 $1, %ymm5, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
-; X64-AVX1-NEXT: vpsrlw $4, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpshufb %xmm10, %xmm9, %xmm10
-; X64-AVX1-NEXT: vpaddb %xmm11, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm5, %xmm8, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
+; X64-AVX1-NEXT: vpand %xmm4, %xmm9, %xmm4
+; X64-AVX1-NEXT: vpshufb %xmm4, %xmm10, %xmm4
+; X64-AVX1-NEXT: vpaddb %xmm4, %xmm12, %xmm4
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm8, %xmm4
+; X64-AVX1-NEXT: vpackusdw %xmm11, %xmm4, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm5, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
+; X64-AVX1-NEXT: vpsrlw $4, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpshufb %xmm11, %xmm10, %xmm11
+; X64-AVX1-NEXT: vpaddb %xmm12, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpsadbw %xmm8, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm5, %xmm9, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
; X64-AVX1-NEXT: vpsrlw $4, %xmm5, %xmm5
-; X64-AVX1-NEXT: vpand %xmm5, %xmm8, %xmm5
-; X64-AVX1-NEXT: vpshufb %xmm5, %xmm9, %xmm5
-; X64-AVX1-NEXT: vpaddb %xmm5, %xmm11, %xmm5
-; X64-AVX1-NEXT: vpaddb %xmm5, %xmm10, %xmm5
-; X64-AVX1-NEXT: vextractf128 $1, %ymm6, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
-; X64-AVX1-NEXT: vpsrlw $4, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpshufb %xmm10, %xmm9, %xmm10
-; X64-AVX1-NEXT: vpaddb %xmm11, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm6, %xmm8, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
+; X64-AVX1-NEXT: vpand %xmm5, %xmm9, %xmm5
+; X64-AVX1-NEXT: vpshufb %xmm5, %xmm10, %xmm5
+; X64-AVX1-NEXT: vpaddb %xmm5, %xmm12, %xmm5
+; X64-AVX1-NEXT: vpsadbw %xmm5, %xmm8, %xmm5
+; X64-AVX1-NEXT: vpackusdw %xmm11, %xmm5, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm6, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
+; X64-AVX1-NEXT: vpsrlw $4, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpshufb %xmm11, %xmm10, %xmm11
+; X64-AVX1-NEXT: vpaddb %xmm12, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpsadbw %xmm8, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm6, %xmm9, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
; X64-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpand %xmm6, %xmm8, %xmm6
-; X64-AVX1-NEXT: vpshufb %xmm6, %xmm9, %xmm6
-; X64-AVX1-NEXT: vpaddb %xmm6, %xmm11, %xmm6
-; X64-AVX1-NEXT: vpaddb %xmm6, %xmm10, %xmm6
-; X64-AVX1-NEXT: vextractf128 $1, %ymm7, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
-; X64-AVX1-NEXT: vpsrlw $4, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm8, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpshufb %xmm10, %xmm9, %xmm10
-; X64-AVX1-NEXT: vpaddb %xmm11, %xmm10, %xmm10
-; X64-AVX1-NEXT: vpand %xmm7, %xmm8, %xmm11
-; X64-AVX1-NEXT: vpshufb %xmm11, %xmm9, %xmm11
+; X64-AVX1-NEXT: vpand %xmm6, %xmm9, %xmm6
+; X64-AVX1-NEXT: vpshufb %xmm6, %xmm10, %xmm6
+; X64-AVX1-NEXT: vpaddb %xmm6, %xmm12, %xmm6
+; X64-AVX1-NEXT: vpsadbw %xmm6, %xmm8, %xmm6
+; X64-AVX1-NEXT: vpackusdw %xmm11, %xmm6, %xmm6
+; X64-AVX1-NEXT: vextractf128 $1, %ymm7, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
+; X64-AVX1-NEXT: vpsrlw $4, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm9, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpshufb %xmm11, %xmm10, %xmm11
+; X64-AVX1-NEXT: vpaddb %xmm12, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpsadbw %xmm8, %xmm11, %xmm11
+; X64-AVX1-NEXT: vpand %xmm7, %xmm9, %xmm12
+; X64-AVX1-NEXT: vpshufb %xmm12, %xmm10, %xmm12
; X64-AVX1-NEXT: vpsrlw $4, %xmm7, %xmm7
-; X64-AVX1-NEXT: vpand %xmm7, %xmm8, %xmm7
-; X64-AVX1-NEXT: vpshufb %xmm7, %xmm9, %xmm7
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm11, %xmm7
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm10, %xmm7
-; X64-AVX1-NEXT: vpxor %xmm8, %xmm8, %xmm8
+; X64-AVX1-NEXT: vpand %xmm7, %xmm9, %xmm7
+; X64-AVX1-NEXT: vpshufb %xmm7, %xmm10, %xmm7
+; X64-AVX1-NEXT: vpaddb %xmm7, %xmm12, %xmm7
+; X64-AVX1-NEXT: vpsadbw %xmm7, %xmm8, %xmm7
+; X64-AVX1-NEXT: vpackusdw %xmm11, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpackusdw %xmm8, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackuswb %xmm8, %xmm0, %xmm0
; X64-AVX1-NEXT: vpsadbw %xmm0, %xmm8, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm0, %xmm9, %xmm0
+; X64-AVX1-NEXT: vpackusdw %xmm8, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm8, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm8, %xmm1
+; X64-AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-AVX1-NEXT: vpackusdw %xmm8, %xmm2, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm8, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm8, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm8, %xmm3, %xmm2
+; X64-AVX1-NEXT: vpackuswb %xmm8, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm8, %xmm2
+; X64-AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X64-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; X64-AVX1-NEXT: vpackusdw %xmm8, %xmm4, %xmm1
+; X64-AVX1-NEXT: vpackuswb %xmm8, %xmm1, %xmm1
; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm8, %xmm1
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm9, %xmm1
+; X64-AVX1-NEXT: vpackusdw %xmm8, %xmm5, %xmm2
+; X64-AVX1-NEXT: vpackuswb %xmm8, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm8, %xmm2
+; X64-AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X64-AVX1-NEXT: vpackusdw %xmm8, %xmm6, %xmm2
+; X64-AVX1-NEXT: vpackuswb %xmm8, %xmm2, %xmm2
; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm8, %xmm2
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm2[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm9, %xmm2
+; X64-AVX1-NEXT: vpackusdw %xmm8, %xmm7, %xmm3
+; X64-AVX1-NEXT: vpackuswb %xmm8, %xmm3, %xmm3
; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm8, %xmm3
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm3[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm3, %xmm9, %xmm3
-; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm8, %xmm4
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm4[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm4, %xmm9, %xmm4
-; X64-AVX1-NEXT: vpsadbw %xmm5, %xmm8, %xmm5
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm5[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm5, %xmm9, %xmm5
-; X64-AVX1-NEXT: vpsadbw %xmm6, %xmm8, %xmm6
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm9 = xmm6[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm6, %xmm9, %xmm6
-; X64-AVX1-NEXT: vpsadbw %xmm7, %xmm8, %xmm7
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm7, %xmm8, %xmm7
-; X64-AVX1-NEXT: vmovd %xmm1, %eax
-; X64-AVX1-NEXT: vmovd %xmm2, %ecx
-; X64-AVX1-NEXT: vmovd %xmm3, %edx
-; X64-AVX1-NEXT: vmovd %xmm5, %esi
-; X64-AVX1-NEXT: vmovd %xmm6, %edi
-; X64-AVX1-NEXT: vmovd %xmm7, %r8d
-; X64-AVX1-NEXT: vpinsrd $1, %esi, %xmm4, %xmm1
-; X64-AVX1-NEXT: vpinsrd $2, %edi, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpinsrd $3, %r8d, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; X64-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
; X64-AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
; X64-AVX1-NEXT: retq
;
@@ -5003,130 +5428,129 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; X86-AVX2-NEXT: pushl %ebp
; X86-AVX2-NEXT: movl %esp, %ebp
; X86-AVX2-NEXT: andl $-32, %esp
-; X86-AVX2-NEXT: subl $192, %esp
+; X86-AVX2-NEXT: subl $160, %esp
; X86-AVX2-NEXT: vmovdqa 40(%ebp), %ymm6
-; X86-AVX2-NEXT: vmovdqa 8(%ebp), %ymm5
-; X86-AVX2-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-AVX2-NEXT: vpand %ymm3, %ymm0, %ymm7
-; X86-AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X86-AVX2-NEXT: # ymm4 = mem[0,1,0,1]
-; X86-AVX2-NEXT: vpshufb %ymm7, %ymm4, %ymm7
+; X86-AVX2-NEXT: vmovdqa 8(%ebp), %ymm3
+; X86-AVX2-NEXT: vpbroadcastb {{.*#+}} ymm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm7
+; X86-AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX2-NEXT: # ymm5 = mem[0,1,0,1]
+; X86-AVX2-NEXT: vpshufb %ymm7, %ymm5, %ymm7
; X86-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
+; X86-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm5, %ymm0
; X86-AVX2-NEXT: vpaddb %ymm7, %ymm0, %ymm0
-; X86-AVX2-NEXT: vmovdqa %ymm0, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
-; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm0
-; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
+; X86-AVX2-NEXT: vmovdqa %ymm0, (%esp) # 32-byte Spill
+; X86-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm5, %ymm0
; X86-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
+; X86-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpshufb %ymm1, %ymm5, %ymm1
+; X86-AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm7
+; X86-AVX2-NEXT: vpand %ymm4, %ymm2, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm5, %ymm0
+; X86-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm1
+; X86-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpshufb %ymm1, %ymm5, %ymm1
; X86-AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm0
; X86-AVX2-NEXT: vmovdqa %ymm0, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm0
-; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
-; X86-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
-; X86-AVX2-NEXT: vpaddb %ymm0, %ymm2, %ymm0
-; X86-AVX2-NEXT: vmovdqa %ymm0, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
-; X86-AVX2-NEXT: vpand %ymm3, %ymm5, %ymm2
-; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
-; X86-AVX2-NEXT: vpsrlw $4, %ymm5, %ymm5
-; X86-AVX2-NEXT: vpand %ymm3, %ymm5, %ymm5
-; X86-AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm5
-; X86-AVX2-NEXT: vpaddb %ymm2, %ymm5, %ymm5
-; X86-AVX2-NEXT: vpand %ymm3, %ymm6, %ymm2
-; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
-; X86-AVX2-NEXT: vpsrlw $4, %ymm6, %ymm6
-; X86-AVX2-NEXT: vpand %ymm3, %ymm6, %ymm6
-; X86-AVX2-NEXT: vpshufb %ymm6, %ymm4, %ymm6
-; X86-AVX2-NEXT: vpaddb %ymm2, %ymm6, %ymm0
-; X86-AVX2-NEXT: vmovdqa %ymm0, (%esp) # 32-byte Spill
-; X86-AVX2-NEXT: vmovdqa 72(%ebp), %ymm2
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm7
-; X86-AVX2-NEXT: vpshufb %ymm7, %ymm4, %ymm7
-; X86-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
-; X86-AVX2-NEXT: vpaddb %ymm7, %ymm2, %ymm6
-; X86-AVX2-NEXT: vmovdqa 104(%ebp), %ymm2
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm1
-; X86-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
-; X86-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
-; X86-AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm7
-; X86-AVX2-NEXT: vmovdqa 136(%ebp), %ymm1
-; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm0
-; X86-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
-; X86-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
+; X86-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm5, %ymm0
+; X86-AVX2-NEXT: vpsrlw $4, %ymm3, %ymm1
+; X86-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpshufb %ymm1, %ymm5, %ymm1
; X86-AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm0
; X86-AVX2-NEXT: vmovdqa %ymm0, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
-; X86-AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
-; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
-; X86-AVX2-NEXT: vpsadbw %ymm0, %ymm5, %ymm5
-; X86-AVX2-NEXT: vpsadbw (%esp), %ymm0, %ymm4 # 32-byte Folded Reload
-; X86-AVX2-NEXT: vpsadbw %ymm0, %ymm6, %ymm6
-; X86-AVX2-NEXT: vpsadbw %ymm0, %ymm7, %ymm7
-; X86-AVX2-NEXT: vmovdqa %ymm7, (%esp) # 32-byte Spill
-; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
+; X86-AVX2-NEXT: vpand %ymm4, %ymm6, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm5, %ymm0
+; X86-AVX2-NEXT: vpsrlw $4, %ymm6, %ymm1
+; X86-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpshufb %ymm1, %ymm5, %ymm1
+; X86-AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm0
; X86-AVX2-NEXT: vmovdqa %ymm0, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
-; X86-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm0
-; X86-AVX2-NEXT: vpaddq %xmm0, %xmm1, %xmm0
-; X86-AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-AVX2-NEXT: vextracti128 $1, %ymm2, %xmm0
-; X86-AVX2-NEXT: vpaddq %xmm0, %xmm2, %xmm0
-; X86-AVX2-NEXT: vextracti128 $1, %ymm3, %xmm1
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm3, %xmm3
-; X86-AVX2-NEXT: vextracti128 $1, %ymm5, %xmm2
-; X86-AVX2-NEXT: vpaddq %xmm2, %xmm5, %xmm7
-; X86-AVX2-NEXT: vextracti128 $1, %ymm4, %xmm2
-; X86-AVX2-NEXT: vpaddq %xmm2, %xmm4, %xmm1
-; X86-AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-AVX2-NEXT: vextracti128 $1, %ymm6, %xmm2
-; X86-AVX2-NEXT: vpaddq %xmm2, %xmm6, %xmm6
-; X86-AVX2-NEXT: vmovdqa (%esp), %ymm1 # 32-byte Reload
-; X86-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; X86-AVX2-NEXT: vpaddq %xmm2, %xmm1, %xmm4
-; X86-AVX2-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %ymm1 # 32-byte Reload
-; X86-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; X86-AVX2-NEXT: vpaddq %xmm2, %xmm1, %xmm2
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm5, %xmm0, %xmm1
-; X86-AVX2-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm5, %xmm0, %xmm5
-; X86-AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm5, %xmm7, %xmm5
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm7, %xmm3, %xmm1
-; X86-AVX2-NEXT: vpbroadcastd %xmm5, %xmm5
-; X86-AVX2-NEXT: vpbroadcastd %xmm1, %xmm1
-; X86-AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
-; X86-AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm6, %xmm1
-; X86-AVX2-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm3 # 16-byte Reload
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm5, %xmm3, %xmm3
-; X86-AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm3, %xmm4, %xmm3
-; X86-AVX2-NEXT: vpbroadcastd %xmm2, %xmm2
-; X86-AVX2-NEXT: vpbroadcastd %xmm3, %xmm3
-; X86-AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X86-AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; X86-AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; X86-AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
-; X86-AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
+; X86-AVX2-NEXT: vmovdqa 72(%ebp), %ymm0
+; X86-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm1
+; X86-AVX2-NEXT: vpshufb %ymm1, %ymm5, %ymm1
+; X86-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm5, %ymm0
+; X86-AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vmovdqa 104(%ebp), %ymm1
+; X86-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm2
+; X86-AVX2-NEXT: vpshufb %ymm2, %ymm5, %ymm2
+; X86-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpshufb %ymm1, %ymm5, %ymm1
+; X86-AVX2-NEXT: vpaddb %ymm2, %ymm1, %ymm2
+; X86-AVX2-NEXT: vmovdqa 136(%ebp), %ymm1
+; X86-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm3
+; X86-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpshufb %ymm3, %ymm5, %ymm3
+; X86-AVX2-NEXT: vpshufb %ymm1, %ymm5, %ymm1
+; X86-AVX2-NEXT: vpaddb %ymm3, %ymm1, %ymm4
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw (%esp), %ymm1, %ymm3 # 32-byte Folded Reload
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm3, %ymm3
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm3, %ymm3
+; X86-AVX2-NEXT: vextracti128 $1, %ymm3, %xmm5
+; X86-AVX2-NEXT: vpackuswb %xmm5, %xmm3, %xmm3
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm7, %ymm5
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm5, %ymm5
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm5 = ymm5[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm5, %ymm5
+; X86-AVX2-NEXT: vextracti128 $1, %ymm5, %xmm6
+; X86-AVX2-NEXT: vpackuswb %xmm6, %xmm5, %xmm5
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[0,2,1,3]
+; X86-AVX2-NEXT: vpxor %xmm6, %xmm6, %xmm6
+; X86-AVX2-NEXT: vpsadbw %xmm6, %xmm3, %xmm3
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm5[0,2,1,3]
+; X86-AVX2-NEXT: vpsadbw %xmm6, %xmm5, %xmm5
+; X86-AVX2-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
+; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm1, %ymm6 # 32-byte Folded Reload
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm3, %ymm3
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm3, %ymm3
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm6, %ymm6
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm6, %ymm6
+; X86-AVX2-NEXT: vpackuswb %ymm6, %ymm3, %ymm3
+; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm1, %ymm6 # 32-byte Folded Reload
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm6, %ymm6
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm6, %ymm6
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackuswb %ymm0, %ymm6, %ymm0
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm4, %ymm4
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm4, %ymm4
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
+; X86-AVX2-NEXT: vpackusdw %ymm1, %ymm4, %ymm4
+; X86-AVX2-NEXT: vpackuswb %ymm4, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; X86-AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,2,1,3,4,6,5,7]
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,4,0,4,0,4,0,4]
+; X86-AVX2-NEXT: vpermd %ymm2, %ymm4, %ymm2
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
+; X86-AVX2-NEXT: vpshufd {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7]
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpermd %ymm0, %ymm4, %ymm0
+; X86-AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm2[6,7]
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm3[0,2,1,3]
+; X86-AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,2,1,3,4,6,5,7]
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm1
+; X86-AVX2-NEXT: vpermd %ymm1, %ymm4, %ymm1
+; X86-AVX2-NEXT: vpblendd {{.*#+}} xmm1 = xmm5[0,1],xmm1[2,3]
+; X86-AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
; X86-AVX2-NEXT: movl %ebp, %esp
; X86-AVX2-NEXT: popl %ebp
; X86-AVX2-NEXT: retl
@@ -5141,103 +5565,111 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; X64-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
; X64-AVX2-NEXT: vpand %ymm0, %ymm8, %ymm0
; X64-AVX2-NEXT: vpshufb %ymm0, %ymm9, %ymm0
-; X64-AVX2-NEXT: vpaddb %ymm0, %ymm10, %ymm0
-; X64-AVX2-NEXT: vpxor %xmm10, %xmm10, %xmm10
-; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm10, %ymm0
-; X64-AVX2-NEXT: vpand %ymm1, %ymm8, %ymm11
-; X64-AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm11
+; X64-AVX2-NEXT: vpaddb %ymm0, %ymm10, %ymm10
+; X64-AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm10, %ymm11
+; X64-AVX2-NEXT: vpand %ymm1, %ymm8, %ymm10
+; X64-AVX2-NEXT: vpshufb %ymm10, %ymm9, %ymm10
; X64-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
; X64-AVX2-NEXT: vpand %ymm1, %ymm8, %ymm1
; X64-AVX2-NEXT: vpshufb %ymm1, %ymm9, %ymm1
-; X64-AVX2-NEXT: vpaddb %ymm1, %ymm11, %ymm1
-; X64-AVX2-NEXT: vpsadbw %ymm1, %ymm10, %ymm1
-; X64-AVX2-NEXT: vpand %ymm2, %ymm8, %ymm11
-; X64-AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm11
+; X64-AVX2-NEXT: vpaddb %ymm1, %ymm10, %ymm1
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm1, %ymm10
+; X64-AVX2-NEXT: vpand %ymm2, %ymm8, %ymm1
+; X64-AVX2-NEXT: vpshufb %ymm1, %ymm9, %ymm1
; X64-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
; X64-AVX2-NEXT: vpand %ymm2, %ymm8, %ymm2
; X64-AVX2-NEXT: vpshufb %ymm2, %ymm9, %ymm2
-; X64-AVX2-NEXT: vpaddb %ymm2, %ymm11, %ymm2
-; X64-AVX2-NEXT: vpsadbw %ymm2, %ymm10, %ymm2
-; X64-AVX2-NEXT: vpand %ymm3, %ymm8, %ymm11
-; X64-AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm11
+; X64-AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm1
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpand %ymm3, %ymm8, %ymm2
+; X64-AVX2-NEXT: vpshufb %ymm2, %ymm9, %ymm2
; X64-AVX2-NEXT: vpsrlw $4, %ymm3, %ymm3
; X64-AVX2-NEXT: vpand %ymm3, %ymm8, %ymm3
; X64-AVX2-NEXT: vpshufb %ymm3, %ymm9, %ymm3
-; X64-AVX2-NEXT: vpaddb %ymm3, %ymm11, %ymm3
-; X64-AVX2-NEXT: vpsadbw %ymm3, %ymm10, %ymm3
-; X64-AVX2-NEXT: vpand %ymm4, %ymm8, %ymm11
-; X64-AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm11
+; X64-AVX2-NEXT: vpaddb %ymm2, %ymm3, %ymm2
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpand %ymm4, %ymm8, %ymm3
+; X64-AVX2-NEXT: vpshufb %ymm3, %ymm9, %ymm3
; X64-AVX2-NEXT: vpsrlw $4, %ymm4, %ymm4
; X64-AVX2-NEXT: vpand %ymm4, %ymm8, %ymm4
; X64-AVX2-NEXT: vpshufb %ymm4, %ymm9, %ymm4
-; X64-AVX2-NEXT: vpaddb %ymm4, %ymm11, %ymm4
-; X64-AVX2-NEXT: vpsadbw %ymm4, %ymm10, %ymm4
-; X64-AVX2-NEXT: vpand %ymm5, %ymm8, %ymm11
-; X64-AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm11
+; X64-AVX2-NEXT: vpaddb %ymm3, %ymm4, %ymm3
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm3, %ymm3
+; X64-AVX2-NEXT: vpand %ymm5, %ymm8, %ymm4
+; X64-AVX2-NEXT: vpshufb %ymm4, %ymm9, %ymm4
; X64-AVX2-NEXT: vpsrlw $4, %ymm5, %ymm5
; X64-AVX2-NEXT: vpand %ymm5, %ymm8, %ymm5
; X64-AVX2-NEXT: vpshufb %ymm5, %ymm9, %ymm5
-; X64-AVX2-NEXT: vpaddb %ymm5, %ymm11, %ymm5
-; X64-AVX2-NEXT: vpsadbw %ymm5, %ymm10, %ymm5
-; X64-AVX2-NEXT: vpand %ymm6, %ymm8, %ymm11
-; X64-AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm11
+; X64-AVX2-NEXT: vpaddb %ymm4, %ymm5, %ymm4
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm4, %ymm4
+; X64-AVX2-NEXT: vpand %ymm6, %ymm8, %ymm5
+; X64-AVX2-NEXT: vpshufb %ymm5, %ymm9, %ymm5
; X64-AVX2-NEXT: vpsrlw $4, %ymm6, %ymm6
; X64-AVX2-NEXT: vpand %ymm6, %ymm8, %ymm6
; X64-AVX2-NEXT: vpshufb %ymm6, %ymm9, %ymm6
-; X64-AVX2-NEXT: vpaddb %ymm6, %ymm11, %ymm6
-; X64-AVX2-NEXT: vpsadbw %ymm6, %ymm10, %ymm6
-; X64-AVX2-NEXT: vpand %ymm7, %ymm8, %ymm11
-; X64-AVX2-NEXT: vpshufb %ymm11, %ymm9, %ymm11
+; X64-AVX2-NEXT: vpaddb %ymm5, %ymm6, %ymm5
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm5, %ymm5
+; X64-AVX2-NEXT: vpand %ymm7, %ymm8, %ymm6
+; X64-AVX2-NEXT: vpshufb %ymm6, %ymm9, %ymm6
; X64-AVX2-NEXT: vpsrlw $4, %ymm7, %ymm7
; X64-AVX2-NEXT: vpand %ymm7, %ymm8, %ymm7
; X64-AVX2-NEXT: vpshufb %ymm7, %ymm9, %ymm7
-; X64-AVX2-NEXT: vpaddb %ymm7, %ymm11, %ymm7
-; X64-AVX2-NEXT: vpsadbw %ymm7, %ymm10, %ymm7
-; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm8
-; X64-AVX2-NEXT: vpaddq %xmm0, %xmm8, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm9
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm9, %xmm1
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm2, %xmm10
-; X64-AVX2-NEXT: vpaddq %xmm2, %xmm10, %xmm2
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm2[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm3, %xmm11
-; X64-AVX2-NEXT: vpaddq %xmm3, %xmm11, %xmm3
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm11 = xmm3[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm4, %xmm12
-; X64-AVX2-NEXT: vpaddq %xmm4, %xmm12, %xmm4
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm12 = xmm4[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm5, %xmm13
-; X64-AVX2-NEXT: vpaddq %xmm5, %xmm13, %xmm5
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm13 = xmm5[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm6, %xmm14
-; X64-AVX2-NEXT: vpaddq %xmm6, %xmm14, %xmm6
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm14 = xmm6[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm7, %xmm15
-; X64-AVX2-NEXT: vpaddq %xmm7, %xmm15, %xmm7
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm15 = xmm7[2,3,2,3]
-; X64-AVX2-NEXT: vpaddq %xmm0, %xmm8, %xmm0
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm9, %xmm1
-; X64-AVX2-NEXT: vmovd %xmm1, %eax
-; X64-AVX2-NEXT: vpaddq %xmm2, %xmm10, %xmm1
-; X64-AVX2-NEXT: vmovd %xmm1, %ecx
-; X64-AVX2-NEXT: vpaddq %xmm3, %xmm11, %xmm1
-; X64-AVX2-NEXT: vmovd %xmm1, %edx
-; X64-AVX2-NEXT: vpaddq %xmm4, %xmm12, %xmm1
-; X64-AVX2-NEXT: vpaddq %xmm5, %xmm13, %xmm2
-; X64-AVX2-NEXT: vmovd %xmm2, %esi
-; X64-AVX2-NEXT: vpaddq %xmm6, %xmm14, %xmm2
-; X64-AVX2-NEXT: vmovd %xmm2, %edi
-; X64-AVX2-NEXT: vpaddq %xmm7, %xmm15, %xmm2
-; X64-AVX2-NEXT: vmovd %xmm2, %r8d
-; X64-AVX2-NEXT: vpinsrd $1, %esi, %xmm1, %xmm1
-; X64-AVX2-NEXT: vpinsrd $2, %edi, %xmm1, %xmm1
-; X64-AVX2-NEXT: vpinsrd $3, %r8d, %xmm1, %xmm1
-; X64-AVX2-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
-; X64-AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpaddb %ymm6, %ymm7, %ymm6
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm6, %ymm7
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm11, %ymm6
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm6 = ymm6[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm6, %ymm6
+; X64-AVX2-NEXT: vextracti128 $1, %ymm6, %xmm8
+; X64-AVX2-NEXT: vpackuswb %xmm8, %xmm6, %xmm6
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm6 = xmm6[0,2,1,3]
+; X64-AVX2-NEXT: vpxor %xmm8, %xmm8, %xmm8
+; X64-AVX2-NEXT: vpsadbw %xmm6, %xmm8, %xmm6
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm10, %ymm9
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm9 = ymm9[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm9, %ymm9
+; X64-AVX2-NEXT: vextracti128 $1, %ymm9, %xmm10
+; X64-AVX2-NEXT: vpackuswb %xmm10, %xmm9, %xmm9
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm9[0,2,1,3]
+; X64-AVX2-NEXT: vpsadbw %xmm8, %xmm9, %xmm8
+; X64-AVX2-NEXT: vpunpckldq {{.*#+}} xmm6 = xmm6[0],xmm8[0],xmm6[1],xmm8[1]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpackuswb %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm3, %ymm2
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm4, %ymm3
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm3, %ymm3
+; X64-AVX2-NEXT: vpackuswb %ymm3, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm5, %ymm3
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm3, %ymm3
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm7, %ymm4
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm4 = ymm4[0,2,1,3]
+; X64-AVX2-NEXT: vpackusdw %ymm0, %ymm4, %ymm4
+; X64-AVX2-NEXT: vpackuswb %ymm4, %ymm3, %ymm3
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm3 = ymm3[0,2,1,3]
+; X64-AVX2-NEXT: vpshufd {{.*#+}} ymm3 = ymm3[0,2,1,3,4,6,5,7]
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm3, %ymm3
+; X64-AVX2-NEXT: vpbroadcastq {{.*#+}} ymm4 = [0,4,0,4,0,4,0,4]
+; X64-AVX2-NEXT: vpermd %ymm3, %ymm4, %ymm3
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,1,3]
+; X64-AVX2-NEXT: vpshufd {{.*#+}} ymm2 = ymm2[0,2,1,3,4,6,5,7]
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpermd %ymm2, %ymm4, %ymm2
+; X64-AVX2-NEXT: vpblendd {{.*#+}} ymm2 = ymm2[0,1,2,3,4,5],ymm3[6,7]
+; X64-AVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm1[0,2,1,3]
+; X64-AVX2-NEXT: vpshufd {{.*#+}} ymm1 = ymm1[0,2,1,3,4,6,5,7]
+; X64-AVX2-NEXT: vpsadbw %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT: vpermd %ymm0, %ymm4, %ymm0
+; X64-AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm6[0,1],xmm0[2,3]
+; X64-AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm2[4,5,6,7]
; X64-AVX2-NEXT: retq
;
; AVX512VL-LABEL: reduce_ctpop_v4i64_buildvector_v8i32:
@@ -5302,14 +5734,14 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; AVX512VL-NEXT: vpshufb %ymm7, %ymm10, %ymm7
; AVX512VL-NEXT: vpaddb %ymm7, %ymm11, %ymm7
; AVX512VL-NEXT: vpsadbw %ymm7, %ymm9, %ymm7
-; AVX512VL-NEXT: vpmovqb %ymm0, %xmm0
-; AVX512VL-NEXT: vpmovqb %ymm1, %xmm1
-; AVX512VL-NEXT: vpmovqb %ymm2, %xmm2
-; AVX512VL-NEXT: vpmovqb %ymm3, %xmm3
-; AVX512VL-NEXT: vpmovqb %ymm4, %xmm4
-; AVX512VL-NEXT: vpmovqb %ymm5, %xmm5
-; AVX512VL-NEXT: vpmovqb %ymm6, %xmm6
-; AVX512VL-NEXT: vpmovqb %ymm7, %xmm7
+; AVX512VL-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512VL-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512VL-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512VL-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512VL-NEXT: vpmovqb %zmm4, %xmm4
+; AVX512VL-NEXT: vpmovqb %zmm5, %xmm5
+; AVX512VL-NEXT: vpmovqb %zmm6, %xmm6
+; AVX512VL-NEXT: vpmovqb %zmm7, %xmm7
; AVX512VL-NEXT: vinserti128 $1, %xmm5, %ymm4, %ymm4
; AVX512VL-NEXT: vinserti128 $1, %xmm7, %ymm6, %ymm5
; AVX512VL-NEXT: vinserti64x4 $1, %ymm4, %zmm5, %zmm4
@@ -5336,14 +5768,14 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; AVX512VPOPCNT-NEXT: vpopcntq %ymm5, %ymm5
; AVX512VPOPCNT-NEXT: vpopcntq %ymm6, %ymm6
; AVX512VPOPCNT-NEXT: vpopcntq %ymm7, %ymm7
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm0, %xmm0
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm1, %xmm1
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm2, %xmm2
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm3, %xmm3
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm4, %xmm4
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm5, %xmm5
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm6, %xmm6
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm7, %xmm7
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm1, %xmm1
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm2, %xmm2
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm3, %xmm3
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm4, %xmm4
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm5, %xmm5
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm6, %xmm6
+; AVX512VPOPCNT-NEXT: vpmovqb %zmm7, %xmm7
; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm5, %ymm4, %ymm4
; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm7, %ymm6, %ymm5
; AVX512VPOPCNT-NEXT: vinserti64x4 $1, %ymm4, %zmm5, %zmm4
diff --git a/llvm/test/CodeGen/X86/vector-reduce-fadd-fast.ll b/llvm/test/CodeGen/X86/vector-reduce-fadd-fast.ll
index 51dd3668850c2..cddd5d13cd77b 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-fadd-fast.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-fadd-fast.ll
@@ -14,21 +14,27 @@
define float @test_v2f32(float %a0, <2 x float> %a1) {
; SSE2-LABEL: test_v2f32:
; SSE2: # %bb.0:
-; SSE2-NEXT: movaps %xmm1, %xmm2
-; SSE2-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm1[1,1]
-; SSE2-NEXT: addss %xmm1, %xmm2
-; SSE2-NEXT: addss %xmm2, %xmm0
+; SSE2-NEXT: xorps %xmm2, %xmm2
+; SSE2-NEXT: addps %xmm1, %xmm2
+; SSE2-NEXT: movaps %xmm2, %xmm1
+; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm2[1,1]
+; SSE2-NEXT: addss %xmm2, %xmm1
+; SSE2-NEXT: addss %xmm1, %xmm0
; SSE2-NEXT: retq
;
; SSE41-LABEL: test_v2f32:
; SSE41: # %bb.0:
-; SSE41-NEXT: movshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
-; SSE41-NEXT: addss %xmm1, %xmm2
-; SSE41-NEXT: addss %xmm2, %xmm0
+; SSE41-NEXT: xorps %xmm2, %xmm2
+; SSE41-NEXT: addps %xmm1, %xmm2
+; SSE41-NEXT: movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSE41-NEXT: addss %xmm2, %xmm1
+; SSE41-NEXT: addss %xmm1, %xmm0
; SSE41-NEXT: retq
;
; AVX1-SLOW-LABEL: test_v2f32:
; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1
; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
; AVX1-SLOW-NEXT: vaddss %xmm2, %xmm1, %xmm1
; AVX1-SLOW-NEXT: vaddss %xmm1, %xmm0, %xmm0
@@ -36,12 +42,16 @@ define float @test_v2f32(float %a0, <2 x float> %a1) {
;
; AVX1-FAST-LABEL: test_v2f32:
; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vmovq {{.*#+}} xmm1 = xmm1[0],zero
+; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
; AVX1-FAST-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: retq
;
; AVX2-LABEL: test_v2f32:
; AVX2: # %bb.0:
+; AVX2-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vaddps %xmm2, %xmm1, %xmm1
; AVX2-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
; AVX2-NEXT: vaddss %xmm2, %xmm1, %xmm1
; AVX2-NEXT: vaddss %xmm1, %xmm0, %xmm0
@@ -49,6 +59,8 @@ define float @test_v2f32(float %a0, <2 x float> %a1) {
;
; AVX512-LABEL: test_v2f32:
; AVX512: # %bb.0:
+; AVX512-NEXT: vxorps %xmm2, %xmm2, %xmm2
+; AVX512-NEXT: vaddps %xmm2, %xmm1, %xmm1
; AVX512-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
; AVX512-NEXT: vaddss %xmm2, %xmm1, %xmm1
; AVX512-NEXT: vaddss %xmm1, %xmm0, %xmm0
@@ -155,7 +167,7 @@ define float @test_v8f32(float %a0, <8 x float> %a1) {
; AVX1-FAST-LABEL: test_v8f32:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm2, %xmm1
+; AVX1-FAST-NEXT: vaddps %xmm2, %xmm1, %xmm1
; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
; AVX1-FAST-NEXT: vaddss %xmm1, %xmm0, %xmm0
@@ -235,8 +247,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
; AVX1-FAST-NEXT: vaddps %ymm2, %ymm1, %ymm1
; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm2
; AVX1-FAST-NEXT: vaddps %xmm2, %xmm1, %xmm1
-; AVX1-FAST-NEXT: vshufpd {{.*#+}} xmm2 = xmm1[1,0]
-; AVX1-FAST-NEXT: vaddps %xmm2, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
; AVX1-FAST-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
@@ -258,7 +269,7 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
; AVX512-LABEL: test_v16f32:
; AVX512: # %bb.0:
; AVX512-NEXT: vextractf64x4 $1, %zmm1, %ymm2
-; AVX512-NEXT: vaddps %zmm2, %zmm1, %zmm1
+; AVX512-NEXT: vaddps %ymm2, %ymm1, %ymm1
; AVX512-NEXT: vextractf128 $1, %ymm1, %xmm2
; AVX512-NEXT: vaddps %xmm2, %xmm1, %xmm1
; AVX512-NEXT: vshufpd {{.*#+}} xmm2 = xmm1[1,0]
@@ -279,6 +290,8 @@ define float @test_v16f32(float %a0, <16 x float> %a1) {
define float @test_v2f32_zero(<2 x float> %a0) {
; SSE2-LABEL: test_v2f32_zero:
; SSE2: # %bb.0:
+; SSE2-NEXT: xorps %xmm1, %xmm1
+; SSE2-NEXT: addps %xmm1, %xmm0
; SSE2-NEXT: movaps %xmm0, %xmm1
; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
; SSE2-NEXT: addss %xmm1, %xmm0
@@ -286,29 +299,39 @@ define float @test_v2f32_zero(<2 x float> %a0) {
;
; SSE41-LABEL: test_v2f32_zero:
; SSE41: # %bb.0:
+; SSE41-NEXT: xorps %xmm1, %xmm1
+; SSE41-NEXT: addps %xmm1, %xmm0
; SSE41-NEXT: movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; SSE41-NEXT: addss %xmm1, %xmm0
; SSE41-NEXT: retq
;
; AVX1-SLOW-LABEL: test_v2f32_zero:
; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; AVX1-SLOW-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: retq
;
; AVX1-FAST-LABEL: test_v2f32_zero:
; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: retq
;
; AVX2-LABEL: test_v2f32_zero:
; AVX2: # %bb.0:
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; AVX2-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_v2f32_zero:
; AVX512: # %bb.0:
+; AVX512-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; AVX512-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX512-NEXT: retq
@@ -405,7 +428,7 @@ define float @test_v8f32_zero(<8 x float> %a0) {
; AVX1-FAST-LABEL: test_v8f32_zero:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
@@ -479,8 +502,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
; AVX1-FAST-NEXT: vaddps %ymm1, %ymm0, %ymm0
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
; AVX1-FAST-NEXT: retq
@@ -500,7 +522,7 @@ define float @test_v16f32_zero(<16 x float> %a0) {
; AVX512-LABEL: test_v16f32_zero:
; AVX512: # %bb.0:
; AVX512-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vaddps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vaddps %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX512-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
@@ -520,6 +542,8 @@ define float @test_v16f32_zero(<16 x float> %a0) {
define float @test_v2f32_undef(<2 x float> %a0) {
; SSE2-LABEL: test_v2f32_undef:
; SSE2: # %bb.0:
+; SSE2-NEXT: xorps %xmm1, %xmm1
+; SSE2-NEXT: addps %xmm1, %xmm0
; SSE2-NEXT: movaps %xmm0, %xmm1
; SSE2-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm0[1,1]
; SSE2-NEXT: addss %xmm1, %xmm0
@@ -527,29 +551,39 @@ define float @test_v2f32_undef(<2 x float> %a0) {
;
; SSE41-LABEL: test_v2f32_undef:
; SSE41: # %bb.0:
+; SSE41-NEXT: xorps %xmm1, %xmm1
+; SSE41-NEXT: addps %xmm1, %xmm0
; SSE41-NEXT: movshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; SSE41-NEXT: addss %xmm1, %xmm0
; SSE41-NEXT: retq
;
; AVX1-SLOW-LABEL: test_v2f32_undef:
; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; AVX1-SLOW-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: retq
;
; AVX1-FAST-LABEL: test_v2f32_undef:
; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero
+; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: retq
;
; AVX2-LABEL: test_v2f32_undef:
; AVX2: # %bb.0:
+; AVX2-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; AVX2-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: test_v2f32_undef:
; AVX512: # %bb.0:
+; AVX512-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovshdup {{.*#+}} xmm1 = xmm0[1,1,3,3]
; AVX512-NEXT: vaddss %xmm1, %xmm0, %xmm0
; AVX512-NEXT: retq
@@ -646,7 +680,7 @@ define float @test_v8f32_undef(<8 x float> %a0) {
; AVX1-FAST-LABEL: test_v8f32_undef:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
@@ -720,8 +754,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
; AVX1-FAST-NEXT: vaddps %ymm1, %ymm0, %ymm0
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
-; AVX1-FAST-NEXT: vaddps %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
; AVX1-FAST-NEXT: retq
@@ -741,7 +774,7 @@ define float @test_v16f32_undef(<16 x float> %a0) {
; AVX512-LABEL: test_v16f32_undef:
; AVX512: # %bb.0:
; AVX512-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vaddps %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vaddps %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX512-NEXT: vaddps %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
@@ -820,7 +853,7 @@ define double @test_v4f64(double %a0, <4 x double> %a1) {
; AVX1-FAST-LABEL: test_v4f64:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-FAST-NEXT: vhaddpd %xmm1, %xmm2, %xmm1
+; AVX1-FAST-NEXT: vaddpd %xmm2, %xmm1, %xmm1
; AVX1-FAST-NEXT: vhaddpd %xmm1, %xmm1, %xmm1
; AVX1-FAST-NEXT: vaddsd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
@@ -896,7 +929,7 @@ define double @test_v8f64(double %a0, <8 x double> %a1) {
; AVX512-LABEL: test_v8f64:
; AVX512: # %bb.0:
; AVX512-NEXT: vextractf64x4 $1, %zmm1, %ymm2
-; AVX512-NEXT: vaddpd %zmm2, %zmm1, %zmm1
+; AVX512-NEXT: vaddpd %ymm2, %ymm1, %ymm1
; AVX512-NEXT: vextractf128 $1, %ymm1, %xmm2
; AVX512-NEXT: vaddpd %xmm2, %xmm1, %xmm1
; AVX512-NEXT: vshufpd {{.*#+}} xmm2 = xmm1[1,0]
@@ -911,11 +944,11 @@ define double @test_v8f64(double %a0, <8 x double> %a1) {
define double @test_v16f64(double %a0, <16 x double> %a1) {
; SSE-LABEL: test_v16f64:
; SSE: # %bb.0:
-; SSE-NEXT: addpd %xmm6, %xmm2
; SSE-NEXT: addpd %xmm7, %xmm3
; SSE-NEXT: addpd %xmm5, %xmm1
-; SSE-NEXT: addpd {{[0-9]+}}(%rsp), %xmm4
; SSE-NEXT: addpd %xmm3, %xmm1
+; SSE-NEXT: addpd {{[0-9]+}}(%rsp), %xmm4
+; SSE-NEXT: addpd %xmm6, %xmm2
; SSE-NEXT: addpd %xmm2, %xmm4
; SSE-NEXT: addpd %xmm1, %xmm4
; SSE-NEXT: movapd %xmm4, %xmm1
@@ -966,7 +999,7 @@ define double @test_v16f64(double %a0, <16 x double> %a1) {
; AVX512: # %bb.0:
; AVX512-NEXT: vaddpd %zmm2, %zmm1, %zmm1
; AVX512-NEXT: vextractf64x4 $1, %zmm1, %ymm2
-; AVX512-NEXT: vaddpd %zmm2, %zmm1, %zmm1
+; AVX512-NEXT: vaddpd %ymm2, %ymm1, %ymm1
; AVX512-NEXT: vextractf128 $1, %ymm1, %xmm2
; AVX512-NEXT: vaddpd %xmm2, %xmm1, %xmm1
; AVX512-NEXT: vshufpd {{.*#+}} xmm2 = xmm1[1,0]
@@ -1037,7 +1070,7 @@ define double @test_v4f64_zero(<4 x double> %a0) {
; AVX1-FAST-LABEL: test_v4f64_zero:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vhaddpd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddpd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
; AVX1-FAST-NEXT: retq
@@ -1106,7 +1139,7 @@ define double @test_v8f64_zero(<8 x double> %a0) {
; AVX512-LABEL: test_v8f64_zero:
; AVX512: # %bb.0:
; AVX512-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vaddpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX512-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
@@ -1120,12 +1153,12 @@ define double @test_v8f64_zero(<8 x double> %a0) {
define double @test_v16f64_zero(<16 x double> %a0) {
; SSE-LABEL: test_v16f64_zero:
; SSE: # %bb.0:
-; SSE-NEXT: addpd %xmm6, %xmm2
-; SSE-NEXT: addpd %xmm4, %xmm0
-; SSE-NEXT: addpd %xmm2, %xmm0
; SSE-NEXT: addpd %xmm7, %xmm3
; SSE-NEXT: addpd %xmm5, %xmm1
; SSE-NEXT: addpd %xmm3, %xmm1
+; SSE-NEXT: addpd %xmm6, %xmm2
+; SSE-NEXT: addpd %xmm4, %xmm0
+; SSE-NEXT: addpd %xmm2, %xmm0
; SSE-NEXT: addpd %xmm1, %xmm0
; SSE-NEXT: movapd %xmm0, %xmm1
; SSE-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
@@ -1171,7 +1204,7 @@ define double @test_v16f64_zero(<16 x double> %a0) {
; AVX512: # %bb.0:
; AVX512-NEXT: vaddpd %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vaddpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX512-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
@@ -1241,7 +1274,7 @@ define double @test_v4f64_undef(<4 x double> %a0) {
; AVX1-FAST-LABEL: test_v4f64_undef:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vhaddpd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vhaddpd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vzeroupper
; AVX1-FAST-NEXT: retq
@@ -1310,7 +1343,7 @@ define double @test_v8f64_undef(<8 x double> %a0) {
; AVX512-LABEL: test_v8f64_undef:
; AVX512: # %bb.0:
; AVX512-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vaddpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX512-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
@@ -1324,12 +1357,12 @@ define double @test_v8f64_undef(<8 x double> %a0) {
define double @test_v16f64_undef(<16 x double> %a0) {
; SSE-LABEL: test_v16f64_undef:
; SSE: # %bb.0:
-; SSE-NEXT: addpd %xmm6, %xmm2
-; SSE-NEXT: addpd %xmm4, %xmm0
-; SSE-NEXT: addpd %xmm2, %xmm0
; SSE-NEXT: addpd %xmm7, %xmm3
; SSE-NEXT: addpd %xmm5, %xmm1
; SSE-NEXT: addpd %xmm3, %xmm1
+; SSE-NEXT: addpd %xmm6, %xmm2
+; SSE-NEXT: addpd %xmm4, %xmm0
+; SSE-NEXT: addpd %xmm2, %xmm0
; SSE-NEXT: addpd %xmm1, %xmm0
; SSE-NEXT: movapd %xmm0, %xmm1
; SSE-NEXT: unpckhpd {{.*#+}} xmm1 = xmm1[1],xmm0[1]
@@ -1375,7 +1408,7 @@ define double @test_v16f64_undef(<16 x double> %a0) {
; AVX512: # %bb.0:
; AVX512-NEXT: vaddpd %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextractf64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vaddpd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vaddpd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextractf128 $1, %ymm0, %xmm1
; AVX512-NEXT: vaddpd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vshufpd {{.*#+}} xmm1 = xmm0[1,0]
diff --git a/llvm/test/CodeGen/X86/vector-trunc.ll b/llvm/test/CodeGen/X86/vector-trunc.ll
index 46f770a349d96..65ee6d7109bc2 100644
--- a/llvm/test/CodeGen/X86/vector-trunc.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc.ll
@@ -1927,132 +1927,151 @@ define void @PR34773(ptr %a0, ptr %a1) {
}
define i16 @PR66194(i8 %q) {
-; SSE2-SSSE3-LABEL: PR66194:
-; SSE2-SSSE3: # %bb.0: # %entry
-; SSE2-SSSE3-NEXT: xorl %eax, %eax
-; SSE2-SSSE3-NEXT: xorl %ecx, %ecx
-; SSE2-SSSE3-NEXT: testb %dil, %dil
-; SSE2-SSSE3-NEXT: setne %al
-; SSE2-SSSE3-NEXT: sete %cl
-; SSE2-SSSE3-NEXT: movl %ecx, %edx
-; SSE2-SSSE3-NEXT: shll $16, %edx
-; SSE2-SSSE3-NEXT: orl %eax, %edx
-; SSE2-SSSE3-NEXT: movd %edx, %xmm0
-; SSE2-SSSE3-NEXT: pinsrw $2, %eax, %xmm0
-; SSE2-SSSE3-NEXT: pinsrw $3, %eax, %xmm0
-; SSE2-SSSE3-NEXT: pinsrw $4, %ecx, %xmm0
-; SSE2-SSSE3-NEXT: pinsrw $5, %eax, %xmm0
-; SSE2-SSSE3-NEXT: pinsrw $6, %eax, %xmm0
-; SSE2-SSSE3-NEXT: pinsrw $7, %ecx, %xmm0
-; SSE2-SSSE3-NEXT: pcmpeqd %xmm1, %xmm1
-; SSE2-SSSE3-NEXT: psubw %xmm1, %xmm0
-; SSE2-SSSE3-NEXT: packuswb %xmm0, %xmm0
-; SSE2-SSSE3-NEXT: pxor %xmm1, %xmm1
-; SSE2-SSSE3-NEXT: psadbw %xmm0, %xmm1
-; SSE2-SSSE3-NEXT: movd %xmm1, %eax
-; SSE2-SSSE3-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE2-SSSE3-NEXT: retq
+; SSE2-LABEL: PR66194:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: xorl %eax, %eax
+; SSE2-NEXT: xorl %ecx, %ecx
+; SSE2-NEXT: testb %dil, %dil
+; SSE2-NEXT: setne %al
+; SSE2-NEXT: sete %cl
+; SSE2-NEXT: movd %ecx, %xmm0
+; SSE2-NEXT: movd %eax, %xmm1
+; SSE2-NEXT: movdqa %xmm1, %xmm2
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSE2-NEXT: movl $1, %eax
+; SSE2-NEXT: movd %eax, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSE2-NEXT: pshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7]
+; SSE2-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: psadbw %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: paddq %xmm1, %xmm0
+; SSE2-NEXT: movq %xmm0, %rax
+; SSE2-NEXT: # kill: def $ax killed $ax killed $rax
+; SSE2-NEXT: retq
+;
+; SSSE3-LABEL: PR66194:
+; SSSE3: # %bb.0: # %entry
+; SSSE3-NEXT: xorl %eax, %eax
+; SSSE3-NEXT: xorl %ecx, %ecx
+; SSSE3-NEXT: testb %dil, %dil
+; SSSE3-NEXT: setne %al
+; SSSE3-NEXT: sete %cl
+; SSSE3-NEXT: movd %ecx, %xmm0
+; SSSE3-NEXT: movd %eax, %xmm1
+; SSSE3-NEXT: movdqa %xmm1, %xmm2
+; SSSE3-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1],xmm2[2],xmm0[2],xmm2[3],xmm0[3],xmm2[4],xmm0[4],xmm2[5],xmm0[5],xmm2[6],xmm0[6],xmm2[7],xmm0[7]
+; SSSE3-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSSE3-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; SSSE3-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; SSSE3-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSSE3-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
+; SSSE3-NEXT: movdqa {{.*#+}} xmm0 = [1,1,1,1,1,1,1,1,u,u,u,u,u,u,u,u]
+; SSSE3-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSSE3-NEXT: pxor %xmm1, %xmm1
+; SSSE3-NEXT: psadbw %xmm0, %xmm1
+; SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSSE3-NEXT: paddq %xmm1, %xmm0
+; SSSE3-NEXT: movq %xmm0, %rax
+; SSSE3-NEXT: # kill: def $ax killed $ax killed $rax
+; SSSE3-NEXT: retq
;
; SSE41-LABEL: PR66194:
; SSE41: # %bb.0: # %entry
-; SSE41-NEXT: xorl %eax, %eax
; SSE41-NEXT: xorl %ecx, %ecx
+; SSE41-NEXT: xorl %eax, %eax
; SSE41-NEXT: testb %dil, %dil
-; SSE41-NEXT: setne %al
-; SSE41-NEXT: sete %cl
-; SSE41-NEXT: movd %eax, %xmm0
-; SSE41-NEXT: pinsrb $2, %ecx, %xmm0
-; SSE41-NEXT: pinsrb $4, %eax, %xmm0
-; SSE41-NEXT: pinsrb $6, %eax, %xmm0
+; SSE41-NEXT: setne %cl
+; SSE41-NEXT: sete %al
+; SSE41-NEXT: movl $1, %edx
+; SSE41-NEXT: movd %edx, %xmm0
+; SSE41-NEXT: pinsrb $1, %edx, %xmm0
+; SSE41-NEXT: pinsrb $2, %edx, %xmm0
+; SSE41-NEXT: pinsrb $3, %edx, %xmm0
+; SSE41-NEXT: pinsrb $4, %edx, %xmm0
+; SSE41-NEXT: pinsrb $5, %edx, %xmm0
+; SSE41-NEXT: pinsrb $6, %edx, %xmm0
+; SSE41-NEXT: pinsrb $7, %edx, %xmm0
; SSE41-NEXT: pinsrb $8, %ecx, %xmm0
-; SSE41-NEXT: pinsrb $10, %eax, %xmm0
+; SSE41-NEXT: pinsrb $9, %eax, %xmm0
+; SSE41-NEXT: pinsrb $10, %ecx, %xmm0
+; SSE41-NEXT: pinsrb $11, %ecx, %xmm0
; SSE41-NEXT: pinsrb $12, %eax, %xmm0
+; SSE41-NEXT: pinsrb $13, %ecx, %xmm0
; SSE41-NEXT: pinsrb $14, %ecx, %xmm0
-; SSE41-NEXT: pcmpeqd %xmm1, %xmm1
-; SSE41-NEXT: psubw %xmm1, %xmm0
-; SSE41-NEXT: packuswb %xmm0, %xmm0
+; SSE41-NEXT: pinsrb $15, %eax, %xmm0
; SSE41-NEXT: pxor %xmm1, %xmm1
; SSE41-NEXT: psadbw %xmm0, %xmm1
-; SSE41-NEXT: movd %xmm1, %eax
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE41-NEXT: paddq %xmm1, %xmm0
+; SSE41-NEXT: movq %xmm0, %rax
+; SSE41-NEXT: # kill: def $ax killed $ax killed $rax
; SSE41-NEXT: retq
;
-; AVX1-LABEL: PR66194:
-; AVX1: # %bb.0: # %entry
-; AVX1-NEXT: xorl %eax, %eax
-; AVX1-NEXT: testb %dil, %dil
-; AVX1-NEXT: setne %al
-; AVX1-NEXT: sete %cl
-; AVX1-NEXT: vmovd %eax, %xmm0
-; AVX1-NEXT: vpinsrb $2, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $4, %eax, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $6, %eax, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $8, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $10, %eax, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $12, %eax, %xmm0, %xmm0
-; AVX1-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vpsubw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX1-NEXT: retq
-;
-; AVX2-LABEL: PR66194:
-; AVX2: # %bb.0: # %entry
-; AVX2-NEXT: xorl %eax, %eax
-; AVX2-NEXT: xorl %ecx, %ecx
-; AVX2-NEXT: testb %dil, %dil
-; AVX2-NEXT: setne %al
-; AVX2-NEXT: sete %cl
-; AVX2-NEXT: vmovd %eax, %xmm0
-; AVX2-NEXT: vpinsrw $1, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $3, %eax, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $4, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $5, %eax, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $6, %eax, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $7, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: retq
+; AVX-LABEL: PR66194:
+; AVX: # %bb.0: # %entry
+; AVX-NEXT: testb %dil, %dil
+; AVX-NEXT: setne %al
+; AVX-NEXT: sete %cl
+; AVX-NEXT: movl $1, %edx
+; AVX-NEXT: vmovd %edx, %xmm0
+; AVX-NEXT: vpinsrb $1, %edx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $3, %edx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $4, %edx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $5, %edx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $6, %edx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $7, %edx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $8, %eax, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $9, %ecx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $10, %eax, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $11, %eax, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $12, %ecx, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $13, %eax, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $14, %eax, %xmm0, %xmm0
+; AVX-NEXT: vpinsrb $15, %ecx, %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vmovq %xmm0, %rax
+; AVX-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX-NEXT: retq
;
; AVX512-LABEL: PR66194:
; AVX512: # %bb.0: # %entry
-; AVX512-NEXT: xorl %eax, %eax
-; AVX512-NEXT: xorl %ecx, %ecx
; AVX512-NEXT: testb %dil, %dil
; AVX512-NEXT: setne %al
; AVX512-NEXT: sete %cl
-; AVX512-NEXT: vmovd %eax, %xmm0
-; AVX512-NEXT: vpinsrw $1, %ecx, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $3, %eax, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $4, %ecx, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $5, %eax, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $6, %eax, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $7, %ecx, %xmm0, %xmm0
-; AVX512-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX512-NEXT: movl $1, %edx
+; AVX512-NEXT: vmovd %edx, %xmm0
+; AVX512-NEXT: vpinsrb $1, %edx, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $2, %edx, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $3, %edx, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $4, %edx, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $5, %edx, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $6, %edx, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $7, %edx, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $8, %eax, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $9, %ecx, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $10, %eax, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $11, %eax, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $12, %ecx, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $13, %eax, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $14, %eax, %xmm0, %xmm0
+; AVX512-NEXT: vpinsrb $15, %ecx, %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $rax
; AVX512-NEXT: retq
entry:
%cmp12.i.13 = icmp ne i8 %q, 0
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
index acddf254c2998..bdd15b6fbfbc9 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
@@ -11,12 +11,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
define i32 @add_v4i32(ptr %p) {
; CHECK-LABEL: @add_v4i32(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
-; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP1]], [[RDX_SHUF]]
-; CHECK-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[BIN_RDX4:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF3]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[BIN_RDX4]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[TMP2:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP0]])
; CHECK-NEXT: ret i32 [[TMP2]]
;
entry:
@@ -231,13 +227,8 @@ for.end:
define float @fadd_v4i32(ptr %p) {
; CHECK-LABEL: @fadd_v4i32(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x float>, ptr [[P:%.*]], align 4, !tbaa [[TBAA7:![0-9]+]]
-; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x float> [[TMP1]], <4 x float> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
-; CHECK-NEXT: [[BIN_RDX:%.*]] = fadd fast <4 x float> [[TMP1]], [[RDX_SHUF]]
-; CHECK-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <4 x float> [[BIN_RDX]], <4 x float> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[BIN_RDX4:%.*]] = fadd fast <4 x float> [[BIN_RDX]], [[RDX_SHUF3]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x float> [[BIN_RDX4]], i32 0
-; CHECK-NEXT: [[BIN_RDX5:%.*]] = fadd fast float 4.200000e+01, [[TMP2]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x float>, ptr [[P:%.*]], align 4, !tbaa [[TBAA7:![0-9]+]]
+; CHECK-NEXT: [[BIN_RDX5:%.*]] = tail call fast float @llvm.vector.reduce.fadd.v4f32(float 4.200000e+01, <4 x float> [[TMP0]])
; CHECK-NEXT: ret float [[BIN_RDX5]]
;
entry:
More information about the llvm-commits
mailing list