[llvm] [WIP][X86] Replace custom ADD reduction pattern matching with ISD::VECREDUCE_ADD support (PR #205098)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 29 05:24:38 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/205098
>From d0c4ba608f900dd376295582735474f27fec1526 Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Wed, 20 May 2026 21:30:30 +0100
Subject: [PATCH] [WIP][X86] Replace custom ADD reduction pattern matching with
ISD::VECREDUCE_ADD support
Make the most of various horizontal add patterns (HADD/PMADDWD/PSADBW instructions etc.) by matching from ISD::VECREDUCE_ADD nodes directly instead trying to match to expanded shuffle chains
This also allows us to greatly simplify the X86PartialReduction pass, avoiding the need to match reductions when the middle end can do it for us
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 301 ++-
llvm/lib/Target/X86/X86PartialReduction.cpp | 94 +-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 1 +
.../X86/avx512-intrinsics-fast-isel.ll | 14 +-
.../CodeGen/X86/buildvec-widen-dotproduct.ll | 6 +-
llvm/test/CodeGen/X86/combine-reductions.ll | 23 +-
llvm/test/CodeGen/X86/dpbusd.ll | 8 +-
llvm/test/CodeGen/X86/dpbusd_const.ll | 4 +-
llvm/test/CodeGen/X86/horizontal-sum.ll | 99 +-
llvm/test/CodeGen/X86/insertelement-zero.ll | 38 +-
llvm/test/CodeGen/X86/madd.ll | 186 +-
.../CodeGen/X86/min-legal-vector-width.ll | 12 +-
llvm/test/CodeGen/X86/phaddsub-extract.ll | 193 +-
llvm/test/CodeGen/X86/phaddsub.ll | 12 +-
llvm/test/CodeGen/X86/pr173924.ll | 52 +-
llvm/test/CodeGen/X86/pr62286.ll | 21 +-
llvm/test/CodeGen/X86/sad.ll | 184 +-
llvm/test/CodeGen/X86/vector-compress.ll | 934 ++++----
.../CodeGen/X86/vector-reduce-add-mask.ll | 1464 ++++++------
.../CodeGen/X86/vector-reduce-add-sext.ll | 477 ++--
.../X86/vector-reduce-add-subvector.ll | 103 +-
.../CodeGen/X86/vector-reduce-add-zext.ll | 1873 ++++++++++++---
llvm/test/CodeGen/X86/vector-reduce-add.ll | 444 ++--
llvm/test/CodeGen/X86/vector-reduce-ctpop.ll | 2042 +++++++++--------
llvm/test/CodeGen/X86/vector-trunc.ll | 199 +-
.../X86/vector-reductions-expanded.ll | 8 +-
26 files changed, 5137 insertions(+), 3655 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 0628aa6baaa7e..50354993dc524 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1180,6 +1180,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// SSE41 can use PHMINPOS to perform v16i8/v8i16 minmax reductions.
// Fallback to ReplaceNodeResults for vXi64 reductions on 32-bit targets.
for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::i64}) {
+ setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
@@ -1571,6 +1572,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
@@ -2047,6 +2049,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_ADD, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
@@ -2808,6 +2811,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
ISD::ANY_EXTEND_VECTOR_INREG,
ISD::SIGN_EXTEND_VECTOR_INREG,
ISD::ZERO_EXTEND_VECTOR_INREG,
+ ISD::VECREDUCE_ADD,
ISD::SINT_TO_FP,
ISD::UINT_TO_FP,
ISD::FP_TO_SINT,
@@ -29884,6 +29888,123 @@ static SDValue LowerVECREDUCE(SDValue Op, const X86Subtarget &Subtarget,
return DAG.getExtractVectorElt(DL, ExtractVT, Src, 0);
}
+static SDValue LowerVECREDUCE_HADD(SDValue Op, const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
+ assert(Op.getOpcode() == ISD::VECREDUCE_ADD &&
+ "Only ADD reductions supported");
+
+ // Only use HADD opcodes if they aren't microcoded or minimizes codesize.
+ if (!shouldUseHorizontalOp(/*IsSingleSource=*/true, DAG, Subtarget))
+ return SDValue();
+
+ SDValue Src = Op.getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ EVT SrcSVT = SrcVT.getScalarType();
+ EVT ExtractVT = Op.getValueType();
+ unsigned NumSrcElts = SrcVT.getVectorNumElements();
+ SDLoc DL(Op);
+
+ if (SrcVT.getSizeInBits() < 128 || !isPowerOf2_32(NumSrcElts))
+ return SDValue();
+
+ if (!((SrcSVT == MVT::i16 || SrcSVT == MVT::i32) && Subtarget.hasSSSE3()))
+ return SDValue();
+
+ unsigned BinOpcode = ISD::ADD;
+ unsigned HorizOpcode = X86ISD::HADD;
+
+ while (SrcVT.getSizeInBits() > 128) {
+ SDValue Lo, Hi;
+ std::tie(Lo, Hi) = splitVector(Src, DAG, DL);
+ SrcVT = Lo.getValueType();
+ Src = DAG.getNode(BinOpcode, DL, SrcVT, Lo, Hi);
+ NumSrcElts /= 2;
+ }
+ assert(SrcVT.is128BitVector() && "128-bit reduction expected");
+
+ // Perform horizontal sums until the whole sum is in element 0.
+ unsigned ReductionSteps = Log2_32(NumSrcElts);
+ for (unsigned I = 0; I != ReductionSteps; ++I) {
+ unsigned NumRdxElts = NumSrcElts >> I;
+ APInt HiElts = APInt::getBitsSet(NumSrcElts, NumRdxElts / 2, NumRdxElts);
+ if (DAG.isIdentityElement(BinOpcode, Op->getFlags(), Src, HiElts, 0)) {
+ APInt LoElts = APInt::getLowBitsSet(NumSrcElts, NumRdxElts / 2);
+ Src = peekThroughDemandedElts(Src, LoElts, DAG);
+ } else {
+ Src = DAG.getNode(HorizOpcode, DL, SrcVT, Src, Src);
+ }
+ }
+
+ return DAG.getExtractVectorElt(DL, ExtractVT, Src, 0);
+}
+
+static SDValue LowerVECREDUCE_ADD(SDValue Op, const X86Subtarget &Subtarget,
+ SelectionDAG &DAG) {
+ SDValue Src = Op.getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ EVT SrcSVT = SrcVT.getScalarType();
+ EVT ExtractVT = Op.getValueType();
+ unsigned NumSrcElts = SrcVT.getVectorNumElements();
+ SDLoc DL(Op);
+
+ if (!isPowerOf2_32(NumSrcElts) || NumSrcElts == 1)
+ return LowerVECREDUCE(Op, Subtarget, DAG, /*AllowScalarization=*/true);
+
+ // vXi8 add reduction - sum lo/hi halves then use PSADBW.
+ if (SrcSVT == MVT::i8) {
+ while (SrcVT.getSizeInBits() > 128) {
+ SDValue Lo, Hi;
+ std::tie(Lo, Hi) = splitVector(Src, DAG, DL);
+ SrcVT = Lo.getValueType();
+ Src = DAG.getNode(ISD::ADD, DL, SrcVT, Lo, Hi);
+ }
+ assert(SrcVT == MVT::v16i8 && "v16i8 reduction expected");
+
+ SDValue Hi = DAG.getVectorShuffle(
+ MVT::v16i8, DL, Src, Src,
+ {8, 9, 10, 11, 12, 13, 14, 15, 8, 9, 10, 11, 12, 13, 14, 15});
+ Src = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Src, Hi);
+ Src = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Src,
+ getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
+ Src = DAG.getBitcast(MVT::v16i8, Src);
+ return DAG.getExtractVectorElt(DL, ExtractVT, Src, 0);
+ }
+
+ KnownBits KnownSrc = DAG.computeKnownBits(Src);
+ if (KnownSrc.getMaxValue().ule(255) && SrcSVT.bitsGT(MVT::i8)) {
+ if (Src.getOpcode() == ISD::ZERO_EXTEND &&
+ Src.getOperand(0).getValueSizeInBits() >= 128 &&
+ Src.getOperand(0).getScalarValueSizeInBits() >= 8) {
+ Src = Src.getOperand(0);
+ SrcVT = Src.getValueType();
+ SrcSVT = SrcVT.getScalarType();
+ }
+
+ // Actually build the SAD, split as 128/256/512 bits for SSE/AVX2/AVX512BW.
+ auto PSADBWBuilder = [&Subtarget](SelectionDAG &DAG, const SDLoc &DL,
+ ArrayRef<SDValue> Ops) {
+ MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);
+ SDValue Zero =
+ getZeroVector(Ops[0].getSimpleValueType(), Subtarget, DAG, DL);
+ return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops[0], Zero);
+ };
+
+ if (SrcSVT.bitsLE(MVT::i32)) {
+ MVT ByteVT = MVT::getVectorVT(MVT::i8, SrcVT.getSizeInBits() / 8);
+ MVT SadVT = MVT::getVectorVT(MVT::i64, SrcVT.getSizeInBits() / 64);
+ Src = DAG.getBitcast(ByteVT, Src);
+ Src = SplitOpsAndApply(DAG, Subtarget, DL, SadVT, {Src}, PSADBWBuilder);
+ Src = DAG.getNode(ISD::VECREDUCE_ADD, DL, MVT::i64, Src);
+ return DAG.getZExtOrTrunc(Src, DL, ExtractVT);
+ }
+ }
+
+ if (SDValue HADD = LowerVECREDUCE_HADD(Op, Subtarget, DAG))
+ return HADD;
+
+ return LowerVECREDUCE(Op, Subtarget, DAG, /*AllowScalarization=*/true);
+}
+
static SDValue lowerAddSub(SDValue Op, SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
MVT VT = Op.getSimpleValueType();
@@ -34740,6 +34861,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR: return LowerVECREDUCE(Op, Subtarget, DAG, true);
+ case ISD::VECREDUCE_ADD: return LowerVECREDUCE_ADD(Op, Subtarget, DAG);
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
case ISD::FMINIMUMNUM:
@@ -35542,6 +35664,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
}
return;
}
+ case ISD::VECREDUCE_ADD: {
+ assert(N->getValueType(0) == MVT::i64 && "Unexpected vector reduction");
+ if (SDValue Res = LowerVECREDUCE(SDValue(N, 0), Subtarget, DAG, true))
+ Results.push_back(Res);
+ return;
+ }
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_SMIN:
case ISD::VECREDUCE_UMAX:
@@ -47464,25 +47592,26 @@ static SDValue combinePredicateReduction(SDNode *Extract, SelectionDAG &DAG,
return DAG.getNegative(Zext, DL, ExtractVT);
}
-static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
+static SDValue combineVPDPBUSDPattern(SDNode *N, const SDLoc &DL,
+ SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
if (!Subtarget.hasVNNI() && !Subtarget.hasAVXVNNI())
return SDValue();
- EVT ExtractVT = Extract->getValueType(0);
- // Verify the type we're extracting is i32, as the output element type of
+ if (N->getOpcode() != ISD::VECREDUCE_ADD)
+ return SDValue();
+
+ EVT ExtractVT = N->getValueType(0);
+ // Verify the type we're reducing to is i32, as the output element type of
// vpdpbusd is i32.
if (ExtractVT != MVT::i32)
return SDValue();
- EVT VT = Extract->getOperand(0).getValueType();
+ SDValue Root = N->getOperand(0);
+ EVT VT = Root.getValueType();
if (!isPowerOf2_32(VT.getVectorNumElements()))
return SDValue();
- // Match shuffle + add pyramid.
- ISD::NodeType BinOp;
- SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
-
// We can't combine to vpdpbusd for zext, because each of the 4 multiplies
// done by vpdpbusd compute a signed 16-bit product that will be sign extended
// before adding into the accumulator.
@@ -47494,7 +47623,7 @@ static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
// Root = Root.getOperand(0);
// If there was a match, we want Root to be a mul.
- if (!Root || Root.getOpcode() != ISD::MUL)
+ if (Root.getOpcode() != ISD::MUL)
return SDValue();
// Check whether we have an extend and mul pattern
@@ -47503,7 +47632,6 @@ static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
return SDValue();
// Create the dot product instruction.
- SDLoc DL(Extract);
unsigned StageBias;
SDValue DP = createVPDPBUSD(DAG, LHS, RHS, StageBias, DL, Subtarget);
@@ -47530,12 +47658,11 @@ static SDValue combineVPDPBUSDPattern(SDNode *Extract, SelectionDAG &DAG,
EVT ResVT =
EVT::getVectorVT(*DAG.getContext(), ExtractVT,
DpVT.getSizeInBits() / ExtractVT.getSizeInBits());
- DP = DAG.getBitcast(ResVT, DP);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, DP,
- Extract->getOperand(1));
+ return DAG.getExtractVectorElt(DL, ExtractVT, DAG.getBitcast(ResVT, DP), 0);
}
-static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
+static SDValue combineBasicSADPattern(SDNode *N, const SDLoc &DL,
+ SelectionDAG &DAG,
const X86Subtarget &Subtarget) {
using namespace SDPatternMatch;
@@ -47543,21 +47670,19 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
if (!Subtarget.hasSSE2())
return SDValue();
- EVT ExtractVT = Extract->getValueType(0);
+ if (N->getOpcode() != ISD::VECREDUCE_ADD)
+ return SDValue();
+
+ EVT ExtractVT = N->getValueType(0);
if (ExtractVT != MVT::i8 && ExtractVT != MVT::i16 && ExtractVT != MVT::i32 &&
ExtractVT != MVT::i64)
return SDValue();
- EVT VT = Extract->getOperand(0).getValueType();
+ SDValue Root = N->getOperand(0);
+ EVT VT = Root.getValueType();
if (!isPowerOf2_32(VT.getVectorNumElements()))
return SDValue();
- // Match shuffle + add pyramid.
- ISD::NodeType BinOp;
- SDValue Root = DAG.matchBinOpReduction(Extract, BinOp, {ISD::ADD});
- if (!Root)
- return SDValue();
-
// The operand is expected to be zero extended from i8.
// In order to convert to i64 and above, additional any/zero/sign
// extend is expected.
@@ -47589,7 +47714,6 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
return SDValue();
// Create the SAD instruction.
- SDLoc DL(Extract);
SDValue SAD = createPSADBW(DAG, Src0, Src1, DL, Subtarget);
// If the original vector was wider than 8 elements, sum over the results
@@ -47599,9 +47723,9 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
if (Stages > 3) {
unsigned SadElems = SadVT.getVectorNumElements();
- for(unsigned i = Stages - 3; i > 0; --i) {
+ for (unsigned i = Stages - 3; i > 0; --i) {
SmallVector<int, 16> Mask(SadElems, -1);
- for(unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
+ for (unsigned j = 0, MaskEnd = 1 << (i - 1); j < MaskEnd; ++j)
Mask[j] = MaskEnd + j;
SDValue Shuffle =
@@ -47614,9 +47738,7 @@ static SDValue combineBasicSADPattern(SDNode *Extract, SelectionDAG &DAG,
// Return the lowest ExtractSizeInBits bits.
EVT ResVT = EVT::getVectorVT(*DAG.getContext(), ExtractVT,
SadVT.getSizeInBits() / ExtractSizeInBits);
- SAD = DAG.getBitcast(ResVT, SAD);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, ExtractVT, SAD,
- Extract->getOperand(1));
+ return DAG.getExtractVectorElt(DL, ExtractVT, DAG.getBitcast(ResVT, SAD), 0);
}
// If this extract is from a loaded vector value and will be used as an
@@ -47982,7 +48104,7 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
ISD::NodeType Opc;
SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc,
- {ISD::ADD, ISD::MUL, ISD::FADD}, true);
+ {ISD::MUL, ISD::FADD}, true);
if (!Rdx)
return SDValue();
@@ -47997,7 +48119,6 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
SDLoc DL(ExtElt);
unsigned NumElts = VecVT.getVectorNumElements();
- unsigned EltSizeInBits = VecVT.getScalarSizeInBits();
// Extend v4i8/v8i8 vector to v16i8, with undef upper 64-bits.
auto WidenToV16I8 = [&](SDValue V, bool ZeroExtend) {
@@ -48051,106 +48172,27 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
}
- // vXi8 add reduction - sub 128-bit vector.
- if (VecVT == MVT::v4i8 || VecVT == MVT::v8i8) {
- Rdx = WidenToV16I8(Rdx, true);
- Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
- DAG.getConstant(0, DL, MVT::v16i8));
- Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
- }
-
// Must be a >=128-bit vector with pow2 elements.
if ((VecVT.getSizeInBits() % 128) != 0 || !isPowerOf2_32(NumElts))
return SDValue();
- // vXi8 add reduction - sum lo/hi halves then use PSADBW.
- if (VT == MVT::i8) {
- while (Rdx.getValueSizeInBits() > 128) {
- SDValue Lo, Hi;
- std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
- VecVT = Lo.getValueType();
- Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
- }
- assert(VecVT == MVT::v16i8 && "v16i8 reduction expected");
-
- SDValue Hi = DAG.getVectorShuffle(
- MVT::v16i8, DL, Rdx, Rdx,
- {8, 9, 10, 11, 12, 13, 14, 15, -1, -1, -1, -1, -1, -1, -1, -1});
- Rdx = DAG.getNode(ISD::ADD, DL, MVT::v16i8, Rdx, Hi);
- Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
- getZeroVector(MVT::v16i8, Subtarget, DAG, DL));
- Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
- }
-
- // See if we can use vXi8 PSADBW add reduction for larger zext types.
- // If the source vector values are 0-255, then we can use PSADBW to
- // sum+zext v8i8 subvectors to vXi64, then perform the reduction.
- // TODO: See if its worth avoiding vXi16/i32 truncations?
- if (Opc == ISD::ADD && NumElts >= 4 && EltSizeInBits >= 16 &&
- EltSizeInBits <= 64 && DAG.computeKnownBits(Rdx).getMaxValue().ule(255) &&
- (EltSizeInBits == 16 || Rdx.getOpcode() == ISD::ZERO_EXTEND ||
- Subtarget.hasAVX512())) {
- if (Rdx.getValueType() == MVT::v8i16) {
- Rdx = DAG.getNode(X86ISD::PACKUS, DL, MVT::v16i8, Rdx,
- DAG.getUNDEF(MVT::v8i16));
- } else {
- EVT ByteVT = VecVT.changeVectorElementType(*DAG.getContext(), MVT::i8);
- Rdx = DAG.getNode(ISD::TRUNCATE, DL, ByteVT, Rdx);
- if (ByteVT.getSizeInBits() < 128)
- Rdx = WidenToV16I8(Rdx, true);
- }
-
- // Build the PSADBW, split as 128/256/512 bits for SSE/AVX2/AVX512BW.
- auto PSADBWBuilder = [](SelectionDAG &DAG, const SDLoc &DL,
- ArrayRef<SDValue> Ops) {
- MVT VT = MVT::getVectorVT(MVT::i64, Ops[0].getValueSizeInBits() / 64);
- SDValue Zero = DAG.getConstant(0, DL, Ops[0].getValueType());
- return DAG.getNode(X86ISD::PSADBW, DL, VT, Ops[0], Zero);
- };
- MVT SadVT = MVT::getVectorVT(MVT::i64, Rdx.getValueSizeInBits() / 64);
- Rdx = SplitOpsAndApply(DAG, Subtarget, DL, SadVT, {Rdx}, PSADBWBuilder);
-
- // TODO: We could truncate to vXi16/vXi32 before performing the reduction.
- while (Rdx.getValueSizeInBits() > 128) {
- SDValue Lo, Hi;
- std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
- VecVT = Lo.getValueType();
- Rdx = DAG.getNode(ISD::ADD, DL, VecVT, Lo, Hi);
- }
- assert(Rdx.getValueType() == MVT::v2i64 && "v2i64 reduction expected");
-
- if (NumElts > 8) {
- SDValue RdxHi = DAG.getVectorShuffle(MVT::v2i64, DL, Rdx, Rdx, {1, -1});
- Rdx = DAG.getNode(ISD::ADD, DL, MVT::v2i64, Rdx, RdxHi);
- }
-
- VecVT = MVT::getVectorVT(VT.getSimpleVT(), 128 / VT.getSizeInBits());
- Rdx = DAG.getBitcast(VecVT, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
- }
-
- // Only use (F)HADD opcodes if they aren't microcoded or minimizes codesize.
+ // Only use FHADD opcodes if they aren't microcoded or minimizes codesize.
if (!shouldUseHorizontalOp(true, DAG, Subtarget))
return SDValue();
- unsigned HorizOpcode = Opc == ISD::ADD ? X86ISD::HADD : X86ISD::FHADD;
+ unsigned HorizOpcode = X86ISD::FHADD;
// 256-bit horizontal instructions operate on 128-bit chunks rather than
// across the whole vector, so we need an extract + hop preliminary stage.
// This is the only step where the operands of the hop are not the same value.
// TODO: We could extend this to handle 512-bit or even longer vectors.
- if (((VecVT == MVT::v16i16 || VecVT == MVT::v8i32) && Subtarget.hasSSSE3()) ||
- ((VecVT == MVT::v8f32 || VecVT == MVT::v4f64) && Subtarget.hasSSE3())) {
- unsigned NumElts = VecVT.getVectorNumElements();
- SDValue Hi = extract128BitVector(Rdx, NumElts / 2, DAG, DL);
- SDValue Lo = extract128BitVector(Rdx, 0, DAG, DL);
- Rdx = DAG.getNode(HorizOpcode, DL, Lo.getValueType(), Hi, Lo);
- VecVT = Rdx.getValueType();
+ if (((VecVT == MVT::v8f32 || VecVT == MVT::v4f64) && Subtarget.hasSSE3())) {
+ SDValue Lo, Hi;
+ std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
+ VecVT = Lo.getValueType();
+ Rdx = DAG.getNode(HorizOpcode, DL, VecVT, Hi, Lo);
}
- if (!((VecVT == MVT::v8i16 || VecVT == MVT::v4i32) && Subtarget.hasSSSE3()) &&
- !((VecVT == MVT::v4f32 || VecVT == MVT::v2f64) && Subtarget.hasSSE3()))
+ if (!((VecVT == MVT::v4f32 || VecVT == MVT::v2f64) && Subtarget.hasSSE3()))
return SDValue();
// extract (add (shuf X), X), 0 --> extract (hadd X, X), 0
@@ -48263,20 +48305,11 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
return DAG.getNode(X86ISD::MMX_MOVD2W, dl, MVT::i32,
InputVector.getOperand(0));
- // Check whether this extract is the root of a sum of absolute differences
- // pattern. This has to be done here because we really want it to happen
- // pre-legalization,
- if (SDValue SAD = combineBasicSADPattern(N, DAG, Subtarget))
- return SAD;
-
- if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DAG, Subtarget))
- return VPDPBUSD;
-
// Attempt to replace an all_of/any_of horizontal reduction with a MOVMSK.
if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget))
return Cmp;
- // Attempt to optimize ADD/FADD/MUL reductions with HADD, promotion etc..
+ // Attempt to optimize FADD/MUL reductions with HADD, promotion etc..
if (SDValue V = combineArithReduction(N, DAG, Subtarget))
return V;
@@ -48365,6 +48398,19 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+static SDValue combineVECREDUCE_ADD(SDNode *N, SelectionDAG &DAG,
+ const X86Subtarget &Subtarget) {
+ SDLoc DL(N);
+
+ if (SDValue SAD = combineBasicSADPattern(N, DL, DAG, Subtarget))
+ return SAD;
+
+ if (SDValue VPDPBUSD = combineVPDPBUSDPattern(N, DL, DAG, Subtarget))
+ return VPDPBUSD;
+
+ return SDValue();
+}
+
// Convert (vXiY *ext(vXi1 bitcast(iX))) to extend_in_reg(broadcast(iX)).
// This is more or less the reverse of combineBitcastvxi1.
static SDValue combineToExtendBoolVectorInReg(
@@ -63325,6 +63371,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case X86ISD::VFMULC: return combineFMulcFCMulc(N, DAG, Subtarget);
case ISD::FNEG: return combineFneg(N, DAG, DCI, Subtarget);
case ISD::TRUNCATE: return combineTruncate(N, DAG, Subtarget);
+ case ISD::VECREDUCE_ADD: return combineVECREDUCE_ADD(N, DAG, Subtarget);
case X86ISD::VTRUNC: return combineVTRUNC(N, DAG, DCI);
case X86ISD::VTRUNCS:
case X86ISD::VTRUNCUS: return combineVTRUNCSAT(N, DAG, DCI, Subtarget);
diff --git a/llvm/lib/Target/X86/X86PartialReduction.cpp b/llvm/lib/Target/X86/X86PartialReduction.cpp
index a017ed9f2ff50..412825fa45b1c 100644
--- a/llvm/lib/Target/X86/X86PartialReduction.cpp
+++ b/llvm/lib/Target/X86/X86PartialReduction.cpp
@@ -42,7 +42,7 @@ class X86PartialReduction {
bool run(Function &F);
private:
- bool tryMAddReplacement(Instruction *Op, bool ReduceInOneBB);
+ bool tryMAddReplacement(Instruction *Op);
bool trySADReplacement(Instruction *Op);
};
@@ -106,8 +106,7 @@ static bool matchVPDPBUSDPattern(const X86Subtarget *ST, BinaryOperator *Mul,
return false;
}
-bool X86PartialReduction::tryMAddReplacement(Instruction *Op,
- bool ReduceInOneBB) {
+bool X86PartialReduction::tryMAddReplacement(Instruction *Op) {
if (!ST->hasSSE2())
return false;
@@ -128,9 +127,7 @@ bool X86PartialReduction::tryMAddReplacement(Instruction *Op,
// If the target support VNNI, leave it to ISel to combine reduce operation
// to VNNI instruction.
- // TODO: we can support transforming reduce to VNNI intrinsic for across block
- // in this pass.
- if (ReduceInOneBB && matchVPDPBUSDPattern(ST, Mul, DL))
+ if (matchVPDPBUSDPattern(ST, Mul, DL))
return false;
// LHS and RHS should be only used once or if they are the same then only
@@ -353,67 +350,6 @@ bool X86PartialReduction::trySADReplacement(Instruction *Op) {
return true;
}
-// Walk backwards from the ExtractElementInst and determine if it is the end of
-// a horizontal reduction. Return the input to the reduction if we find one.
-static Value *matchAddReduction(const ExtractElementInst &EE,
- bool &ReduceInOneBB) {
- ReduceInOneBB = true;
- // Make sure we're extracting index 0.
- auto *Index = dyn_cast<ConstantInt>(EE.getIndexOperand());
- if (!Index || !Index->isNullValue())
- return nullptr;
-
- const auto *BO = dyn_cast<BinaryOperator>(EE.getVectorOperand());
- if (!BO || BO->getOpcode() != Instruction::Add || !BO->hasOneUse())
- return nullptr;
- if (EE.getParent() != BO->getParent())
- ReduceInOneBB = false;
-
- unsigned NumElems = cast<FixedVectorType>(BO->getType())->getNumElements();
- // Ensure the reduction size is a power of 2.
- if (!isPowerOf2_32(NumElems))
- return nullptr;
-
- const Value *Op = BO;
- unsigned Stages = Log2_32(NumElems);
- for (unsigned i = 0; i != Stages; ++i) {
- const auto *BO = dyn_cast<BinaryOperator>(Op);
- if (!BO || BO->getOpcode() != Instruction::Add)
- return nullptr;
- if (EE.getParent() != BO->getParent())
- ReduceInOneBB = false;
-
- // If this isn't the first add, then it should only have 2 users, the
- // shuffle and another add which we checked in the previous iteration.
- if (i != 0 && !BO->hasNUses(2))
- return nullptr;
-
- Value *LHS = BO->getOperand(0);
- Value *RHS = BO->getOperand(1);
-
- auto *Shuffle = dyn_cast<ShuffleVectorInst>(LHS);
- if (Shuffle) {
- Op = RHS;
- } else {
- Shuffle = dyn_cast<ShuffleVectorInst>(RHS);
- Op = LHS;
- }
-
- // The first operand of the shuffle should be the same as the other operand
- // of the bin op.
- if (!Shuffle || Shuffle->getOperand(0) != Op)
- return nullptr;
-
- // Verify the shuffle has the expected (at this stage of the pyramid) mask.
- unsigned MaskEnd = 1 << i;
- for (unsigned Index = 0; Index < MaskEnd; ++Index)
- if (Shuffle->getMaskValue(Index) != (int)(MaskEnd + Index))
- return nullptr;
- }
-
- return const_cast<Value *>(Op);
-}
-
// See if this BO is reachable from this Phi by walking forward through single
// use BinaryOperators with the same opcode. If we get back then we know we've
// found a loop and it is safe to step through this Add to find more leaves.
@@ -447,9 +383,7 @@ static void collectLeaves(Value *Root, SmallVectorImpl<Instruction *> &Leaves) {
continue;
if (auto *PN = dyn_cast<PHINode>(V)) {
- // PHI node should have single use unless it is the root node, then it
- // has 2 uses.
- if (!PN->hasNUses(PN == Root ? 2 : 1))
+ if (!PN->hasNUses(1))
break;
// Push incoming values to the worklist.
@@ -461,14 +395,14 @@ static void collectLeaves(Value *Root, SmallVectorImpl<Instruction *> &Leaves) {
if (auto *BO = dyn_cast<BinaryOperator>(V)) {
if (BO->getOpcode() == Instruction::Add) {
// Simple case. Single use, just push its operands to the worklist.
- if (BO->hasNUses(BO == Root ? 2 : 1)) {
+ if (BO->hasNUses(1)) {
append_range(Worklist, BO->operands());
continue;
}
// If there is additional use, make sure it is an unvisited phi that
// gets us back to this node.
- if (BO->hasNUses(BO == Root ? 3 : 2)) {
+ if (BO->hasNUses(2)) {
PHINode *PN = nullptr;
for (auto *U : BO->users())
if (auto *P = dyn_cast<PHINode>(U))
@@ -492,7 +426,7 @@ static void collectLeaves(Value *Root, SmallVectorImpl<Instruction *> &Leaves) {
// Not an add or phi, make it a leaf.
if (auto *I = dyn_cast<Instruction>(V)) {
- if (!V->hasNUses(I == Root ? 2 : 1))
+ if (!V->hasNUses(1))
continue;
// Add this as a leaf.
@@ -508,22 +442,18 @@ bool X86PartialReduction::run(Function &F) {
bool MadeChange = false;
for (auto &BB : F) {
for (auto &I : BB) {
- auto *EE = dyn_cast<ExtractElementInst>(&I);
- if (!EE)
- continue;
-
- bool ReduceInOneBB;
// First find a reduction tree.
- // FIXME: Do we need to handle other opcodes than Add?
- Value *Root = matchAddReduction(*EE, ReduceInOneBB);
- if (!Root)
+ // FIXME: Do we need to handle other reduction opcodes than Add?
+ auto *II = dyn_cast<IntrinsicInst>(&I);
+ if (!II || II->getIntrinsicID() != Intrinsic::vector_reduce_add)
continue;
+ Value *Root = II->getOperand(0);
SmallVector<Instruction *, 8> Leaves;
collectLeaves(Root, Leaves);
for (Instruction *I : Leaves) {
- if (tryMAddReplacement(I, ReduceInOneBB)) {
+ if (tryMAddReplacement(I)) {
MadeChange = true;
continue;
}
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 8838fd7e71f02..b5ee3c746b0ec 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -6855,6 +6855,7 @@ bool X86TTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
switch (II->getIntrinsicID()) {
default:
return true;
+ case Intrinsic::vector_reduce_add:
case Intrinsic::vector_reduce_smax:
case Intrinsic::vector_reduce_smin:
case Intrinsic::vector_reduce_umax:
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
index 1e14256db3584..e5b82c4213410 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
@@ -6519,7 +6519,7 @@ define i64 @test_mm512_reduce_add_epi64(<8 x i64> %__W) nounwind {
; X86-LABEL: test_mm512_reduce_add_epi64:
; X86: # %bb.0: # %entry
; X86-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; X86-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; X86-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; X86-NEXT: vextracti128 $1, %ymm0, %xmm1
; X86-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -6532,7 +6532,7 @@ define i64 @test_mm512_reduce_add_epi64(<8 x i64> %__W) nounwind {
; X64-LABEL: test_mm512_reduce_add_epi64:
; X64: # %bb.0: # %entry
; X64-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; X64-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; X64-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; X64-NEXT: vextracti128 $1, %ymm0, %xmm1
; X64-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -6684,7 +6684,7 @@ define i64 @test_mm512_mask_reduce_add_epi64(i8 zeroext %__M, <8 x i64> %__W) no
; X86-NEXT: kmovw %eax, %k1
; X86-NEXT: vmovdqa64 %zmm0, %zmm0 {%k1} {z}
; X86-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; X86-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; X86-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; X86-NEXT: vextracti128 $1, %ymm0, %xmm1
; X86-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -6699,7 +6699,7 @@ define i64 @test_mm512_mask_reduce_add_epi64(i8 zeroext %__M, <8 x i64> %__W) no
; X64-NEXT: kmovw %edi, %k1
; X64-NEXT: vmovdqa64 %zmm0, %zmm0 {%k1} {z}
; X64-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; X64-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; X64-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; X64-NEXT: vextracti128 $1, %ymm0, %xmm1
; X64-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -6875,7 +6875,7 @@ define i32 @test_mm512_reduce_add_epi32(<8 x i64> %__W) nounwind {
; CHECK-LABEL: test_mm512_reduce_add_epi32:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -6958,7 +6958,7 @@ define i32 @test_mm512_mask_reduce_add_epi32(i16 zeroext %__M, <8 x i64> %__W) n
; X86-NEXT: kmovw %eax, %k1
; X86-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
; X86-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; X86-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; X86-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; X86-NEXT: vextracti128 $1, %ymm0, %xmm1
; X86-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; X86-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -6974,7 +6974,7 @@ define i32 @test_mm512_mask_reduce_add_epi32(i16 zeroext %__M, <8 x i64> %__W) n
; X64-NEXT: kmovw %edi, %k1
; X64-NEXT: vmovdqa32 %zmm0, %zmm0 {%k1} {z}
; X64-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; X64-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; X64-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; X64-NEXT: vextracti128 $1, %ymm0, %xmm1
; X64-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; X64-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll b/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll
index 07c81c6f8b7a4..336b971c23200 100644
--- a/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll
+++ b/llvm/test/CodeGen/X86/buildvec-widen-dotproduct.ll
@@ -417,8 +417,7 @@ define i32 @dot_ext_v2i16_v2i32(ptr %a, i64 %a_stride, ptr %b) nounwind {
; SSE4: # %bb.0:
; SSE4-NEXT: movzwl (%rdi), %eax
; SSE4-NEXT: movd %eax, %xmm0
-; SSE4-NEXT: pinsrw $1, (%rdi,%rsi), %xmm0
-; SSE4-NEXT: pmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; SSE4-NEXT: pinsrw $2, (%rdi,%rsi), %xmm0
; SSE4-NEXT: movd {{.*#+}} xmm1 = mem[0],zero,zero,zero
; SSE4-NEXT: pmovsxwd %xmm1, %xmm1
; SSE4-NEXT: pmulld %xmm0, %xmm1
@@ -431,8 +430,7 @@ define i32 @dot_ext_v2i16_v2i32(ptr %a, i64 %a_stride, ptr %b) nounwind {
; AVX: # %bb.0:
; AVX-NEXT: movzwl (%rdi), %eax
; AVX-NEXT: vmovd %eax, %xmm0
-; AVX-NEXT: vpinsrw $1, (%rdi,%rsi), %xmm0, %xmm0
-; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX-NEXT: vpinsrw $2, (%rdi,%rsi), %xmm0, %xmm0
; AVX-NEXT: vmovd {{.*#+}} xmm1 = mem[0],zero,zero,zero
; AVX-NEXT: vpmovsxwd %xmm1, %xmm1
; AVX-NEXT: vpmulld %xmm0, %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/combine-reductions.ll b/llvm/test/CodeGen/X86/combine-reductions.ll
index 5f59cce91e2a3..912a2caecdc07 100644
--- a/llvm/test/CodeGen/X86/combine-reductions.ll
+++ b/llvm/test/CodeGen/X86/combine-reductions.ll
@@ -90,17 +90,10 @@ define { i16, i16 } @test_reduce_v16i16_with_add(<16 x i16> %x, <16 x i16> %y) {
; SSE41-FAST: # %bb.0:
; SSE41-FAST-NEXT: movdqa %xmm0, %xmm4
; SSE41-FAST-NEXT: paddw %xmm1, %xmm4
-; SSE41-FAST-NEXT: pshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; SSE41-FAST-NEXT: paddw %xmm4, %xmm5
-; SSE41-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm5[1,1,1,1]
-; SSE41-FAST-NEXT: paddw %xmm5, %xmm4
; SSE41-FAST-NEXT: phaddw %xmm4, %xmm4
-; SSE41-FAST-NEXT: movdqa %xmm1, %xmm5
-; SSE41-FAST-NEXT: phaddw %xmm0, %xmm5
-; SSE41-FAST-NEXT: phaddw %xmm5, %xmm5
-; SSE41-FAST-NEXT: phaddw %xmm5, %xmm5
-; SSE41-FAST-NEXT: phaddw %xmm5, %xmm5
-; SSE41-FAST-NEXT: movd %xmm5, %eax
+; SSE41-FAST-NEXT: phaddw %xmm4, %xmm4
+; SSE41-FAST-NEXT: phaddw %xmm4, %xmm4
+; SSE41-FAST-NEXT: movd %xmm4, %eax
; SSE41-FAST-NEXT: pshuflw {{.*#+}} xmm4 = xmm4[0,0,0,0,4,5,6,7]
; SSE41-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,1,0,1]
; SSE41-FAST-NEXT: pcmpeqw %xmm4, %xmm1
@@ -145,18 +138,12 @@ define { i16, i16 } @test_reduce_v16i16_with_add(<16 x i16> %x, <16 x i16> %y) {
; AVX2-FAST-LABEL: test_reduce_v16i16_with_add:
; AVX2-FAST: # %bb.0:
; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm2
-; AVX2-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm3
-; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; AVX2-FAST-NEXT: vpaddw %xmm4, %xmm3, %xmm3
-; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
-; AVX2-FAST-NEXT: vpaddw %xmm4, %xmm3, %xmm3
-; AVX2-FAST-NEXT: vphaddw %xmm3, %xmm3, %xmm3
-; AVX2-FAST-NEXT: vphaddw %xmm0, %xmm2, %xmm2
+; AVX2-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm2
; AVX2-FAST-NEXT: vphaddw %xmm2, %xmm2, %xmm2
; AVX2-FAST-NEXT: vphaddw %xmm2, %xmm2, %xmm2
; AVX2-FAST-NEXT: vphaddw %xmm2, %xmm2, %xmm2
; AVX2-FAST-NEXT: vmovd %xmm2, %eax
-; AVX2-FAST-NEXT: vpbroadcastw %xmm3, %ymm2
+; AVX2-FAST-NEXT: vpbroadcastw %xmm2, %ymm2
; AVX2-FAST-NEXT: vpcmpeqw %ymm2, %ymm0, %ymm0
; AVX2-FAST-NEXT: vpcmpeqd %ymm2, %ymm2, %ymm2
; AVX2-FAST-NEXT: vpxor %ymm2, %ymm0, %ymm0
diff --git a/llvm/test/CodeGen/X86/dpbusd.ll b/llvm/test/CodeGen/X86/dpbusd.ll
index 7bd22d57347b3..d1781fb31ba64 100644
--- a/llvm/test/CodeGen/X86/dpbusd.ll
+++ b/llvm/test/CodeGen/X86/dpbusd.ll
@@ -111,7 +111,7 @@ define i32 @mul_zext(ptr%a, ptr%b, i32 %c, i32 %n) {
; AVXVNNI-AVX512-NEXT: vpmullw %ymm0, %ymm1, %ymm0
; AVXVNNI-AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVXVNNI-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVXVNNI-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVXVNNI-AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVXVNNI-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVXVNNI-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -130,7 +130,7 @@ define i32 @mul_zext(ptr%a, ptr%b, i32 %c, i32 %n) {
; AVX512-NEXT: vpmullw %ymm0, %ymm1, %ymm0
; AVX512-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -184,7 +184,7 @@ define i32 @mul_sext(ptr%a, ptr%b, i32 %c, i32 %n) {
; AVXVNNI-AVX512-NEXT: vpmullw %ymm0, %ymm1, %ymm0
; AVXVNNI-AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
; AVXVNNI-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVXVNNI-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVXVNNI-AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVXVNNI-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVXVNNI-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVXVNNI-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -203,7 +203,7 @@ define i32 @mul_sext(ptr%a, ptr%b, i32 %c, i32 %n) {
; AVX512-NEXT: vpmullw %ymm0, %ymm1, %ymm0
; AVX512-NEXT: vpmovsxwd %ymm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/dpbusd_const.ll b/llvm/test/CodeGen/X86/dpbusd_const.ll
index 1d6c3f7c5c6e8..bb47df59eefad 100644
--- a/llvm/test/CodeGen/X86/dpbusd_const.ll
+++ b/llvm/test/CodeGen/X86/dpbusd_const.ll
@@ -8,8 +8,8 @@ define i32 @mul_4xi8_zc_exceed(<4 x i8> %a, i32 %c) {
; CHECK-LABEL: mul_4xi8_zc_exceed:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; CHECK-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [0,0,1,0,2,0,128,0]
; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; CHECK-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [2,128,2,128]
; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0
@@ -180,8 +180,8 @@ define i32 @mul_4xi8_cs_exceed(<4 x i8> %a, i32 %c) {
; CHECK-LABEL: mul_4xi8_cs_exceed:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vpmovsxbd %xmm0, %xmm0
+; CHECK-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [0,0,1,0,2,0,256,0]
; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; CHECK-NEXT: vpmaddwd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1 # [2,256,2,256]
; CHECK-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; CHECK-NEXT: vpaddd %xmm0, %xmm1, %xmm0
diff --git a/llvm/test/CodeGen/X86/horizontal-sum.ll b/llvm/test/CodeGen/X86/horizontal-sum.ll
index 568150cfa3971..8085abce81b40 100644
--- a/llvm/test/CodeGen/X86/horizontal-sum.ll
+++ b/llvm/test/CodeGen/X86/horizontal-sum.ll
@@ -1042,87 +1042,52 @@ define <4 x i32> @reduction_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32
; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm6
; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm3 = xmm6[1,1,1,1]
; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm4 = xmm4[0],xmm2[0]
; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm6[0],xmm1[1],xmm6[1]
+; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm1
; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
-; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm0
+; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; SSSE3-SLOW-NEXT: retq
;
; SSSE3-FAST-LABEL: reduction_sum_v4i32_v4i32:
; SSSE3-FAST: # %bb.0:
-; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; SSSE3-FAST-NEXT: paddd %xmm4, %xmm0
-; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; SSSE3-FAST-NEXT: paddd %xmm1, %xmm4
-; SSSE3-FAST-NEXT: phaddd %xmm4, %xmm0
-; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; SSSE3-FAST-NEXT: paddd %xmm2, %xmm1
-; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; SSSE3-FAST-NEXT: paddd %xmm3, %xmm2
-; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm1
-; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm2
+; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm0
+; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm0
; SSSE3-FAST-NEXT: retq
;
-; AVX1-SLOW-LABEL: reduction_sum_v4i32_v4i32:
-; AVX1-SLOW: # %bb.0:
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
-; AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm1, %xmm1
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[1,1,1,1]
-; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
-; AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm2, %xmm2
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[1,1,1,1]
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[2,3,2,3]
-; AVX1-SLOW-NEXT: vpaddd %xmm6, %xmm3, %xmm3
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[1,1,1,1]
-; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
-; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm2, %xmm2
-; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
-; AVX1-SLOW-NEXT: retq
+; AVX-SLOW-LABEL: reduction_sum_v4i32_v4i32:
+; AVX-SLOW: # %bb.0:
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vmovd %xmm1, %eax
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; AVX-SLOW-NEXT: vpaddd %xmm1, %xmm2, %xmm1
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vmovd %xmm1, %ecx
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
+; AVX-SLOW-NEXT: vpaddd %xmm1, %xmm3, %xmm1
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vmovd %xmm1, %edx
+; AVX-SLOW-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
+; AVX-SLOW-NEXT: retq
;
; AVX-FAST-LABEL: reduction_sum_v4i32_v4i32:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; AVX-FAST-NEXT: vpaddd %xmm4, %xmm0, %xmm0
-; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
-; AVX-FAST-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm2
; AVX-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0
-; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; AVX-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1
-; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; AVX-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
-; AVX-FAST-NEXT: vphaddd %xmm2, %xmm1, %xmm1
-; AVX-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; AVX-FAST-NEXT: vphaddd %xmm2, %xmm0, %xmm0
; AVX-FAST-NEXT: retq
-;
-; AVX2-SLOW-LABEL: reduction_sum_v4i32_v4i32:
-; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
-; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
-; AVX2-SLOW-NEXT: vpaddd %xmm5, %xmm1, %xmm1
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
-; AVX2-SLOW-NEXT: vpaddd %xmm5, %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
-; AVX2-SLOW-NEXT: vpaddd %xmm5, %xmm3, %xmm3
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm5 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0],xmm1[1],xmm4[2,3]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm4 = xmm4[0,1],xmm5[2,3]
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; AVX2-SLOW-NEXT: vpbroadcastd %xmm3, %xmm1
-; AVX2-SLOW-NEXT: vpbroadcastd %xmm2, %xmm2
-; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
-; AVX2-SLOW-NEXT: retq
%5 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %0)
%6 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %1)
%7 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %2)
diff --git a/llvm/test/CodeGen/X86/insertelement-zero.ll b/llvm/test/CodeGen/X86/insertelement-zero.ll
index e1c8cefa73d8a..4707c2be2911a 100644
--- a/llvm/test/CodeGen/X86/insertelement-zero.ll
+++ b/llvm/test/CodeGen/X86/insertelement-zero.ll
@@ -1,12 +1,12 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=SSE,SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse3 | FileCheck %s --check-prefixes=SSE,SSE3
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSE,SSSE3
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=SSE,SSE41
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown | FileCheck %s --check-prefixes=CHECK,SSE,SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=CHECK,SSE,SSSE3
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse4.1 | FileCheck %s --check-prefixes=CHECK,SSE,SSE41
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx | FileCheck %s --check-prefixes=CHECK,AVX,AVX1
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=CHECK,AVX,AVX2
define <2 x double> @insert_v2f64_z1(<2 x double> %a) {
; SSE2-LABEL: insert_v2f64_z1:
@@ -554,24 +554,10 @@ define <4 x i32> @PR41512_loads(ptr %p1, ptr %p2) {
;
; Expected result is that the add reduction computes the sum 0+0+0+0+0+77+0+77 = 154.
define i64 @fold_insertelement_to_and(i32 noundef %arg) {
-; SSE-LABEL: fold_insertelement_to_and:
-; SSE: # %bb.0:
-; SSE-NEXT: movl $154, %eax
-; SSE-NEXT: retq
-;
-; AVX1-LABEL: fold_insertelement_to_and:
-; AVX1: # %bb.0:
-; AVX1-NEXT: movl $154, %eax
-; AVX1-NEXT: retq
-;
-; AVX2-LABEL: fold_insertelement_to_and:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpmovsxbq {{.*#+}} xmm0 = [0,77]
-; AVX2-NEXT: vpaddq %xmm0, %xmm0, %xmm1
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovq %xmm0, %rax
-; AVX2-NEXT: retq
+; CHECK-LABEL: fold_insertelement_to_and:
+; CHECK: # %bb.0:
+; CHECK-NEXT: movl $154, %eax
+; CHECK-NEXT: retq
%i = shufflevector <8 x i64> zeroinitializer, <8 x i64> splat (i64 77), <8 x i32> <i32 poison, i32 poison, i32 poison, i32 poison, i32 4, i32 8, i32 6, i32 10>
%i1 = insertelement <8 x i64> %i, i64 0, i64 0
%i2 = insertelement <8 x i64> %i1, i64 0, i64 2
diff --git a/llvm/test/CodeGen/X86/madd.ll b/llvm/test/CodeGen/X86/madd.ll
index 4ca0b05dbe92b..a9ed3a79aed97 100644
--- a/llvm/test/CodeGen/X86/madd.ll
+++ b/llvm/test/CodeGen/X86/madd.ll
@@ -316,7 +316,7 @@ define i32 @_Z10test_shortPsS_i_512(ptr nocapture readonly, ptr nocapture readon
; AVX512-NEXT: jne .LBB2_1
; AVX512-NEXT: # %bb.2: # %middle.block
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -358,10 +358,10 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
; SSE-NEXT: movl %edx, %eax
; SSE-NEXT: pxor %xmm0, %xmm0
; SSE-NEXT: xorl %ecx, %ecx
-; SSE-NEXT: pxor %xmm2, %xmm2
; SSE-NEXT: pxor %xmm4, %xmm4
-; SSE-NEXT: pxor %xmm1, %xmm1
; SSE-NEXT: pxor %xmm3, %xmm3
+; SSE-NEXT: pxor %xmm2, %xmm2
+; SSE-NEXT: pxor %xmm1, %xmm1
; SSE-NEXT: .p2align 4
; SSE-NEXT: .LBB3_1: # %vector.body
; SSE-NEXT: # =>This Inner Loop Header: Depth=1
@@ -371,27 +371,27 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
; SSE-NEXT: movdqu 48(%rdi,%rcx,2), %xmm8
; SSE-NEXT: movdqu (%rsi,%rcx,2), %xmm9
; SSE-NEXT: pmaddwd %xmm5, %xmm9
-; SSE-NEXT: paddd %xmm9, %xmm2
+; SSE-NEXT: paddd %xmm9, %xmm4
; SSE-NEXT: movdqu 16(%rsi,%rcx,2), %xmm5
; SSE-NEXT: pmaddwd %xmm6, %xmm5
-; SSE-NEXT: paddd %xmm5, %xmm4
+; SSE-NEXT: paddd %xmm5, %xmm3
; SSE-NEXT: movdqu 32(%rsi,%rcx,2), %xmm5
; SSE-NEXT: pmaddwd %xmm7, %xmm5
-; SSE-NEXT: paddd %xmm5, %xmm1
+; SSE-NEXT: paddd %xmm5, %xmm2
; SSE-NEXT: movdqu 48(%rsi,%rcx,2), %xmm5
; SSE-NEXT: pmaddwd %xmm8, %xmm5
-; SSE-NEXT: paddd %xmm5, %xmm3
+; SSE-NEXT: paddd %xmm5, %xmm1
; SSE-NEXT: addq $16, %rcx
; SSE-NEXT: cmpq %rcx, %rax
; SSE-NEXT: jne .LBB3_1
; SSE-NEXT: # %bb.2: # %middle.block
; SSE-NEXT: paddd %xmm0, %xmm4
-; SSE-NEXT: paddd %xmm0, %xmm3
-; SSE-NEXT: paddd %xmm4, %xmm3
; SSE-NEXT: paddd %xmm0, %xmm2
+; SSE-NEXT: paddd %xmm4, %xmm2
+; SSE-NEXT: paddd %xmm0, %xmm3
; SSE-NEXT: paddd %xmm0, %xmm1
-; SSE-NEXT: paddd %xmm2, %xmm1
; SSE-NEXT: paddd %xmm3, %xmm1
+; SSE-NEXT: paddd %xmm2, %xmm1
; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; SSE-NEXT: paddd %xmm1, %xmm0
; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -429,16 +429,16 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
; AVX1-NEXT: cmpq %rcx, %rax
; AVX1-NEXT: jne .LBB3_1
; AVX1-NEXT: # %bb.2: # %middle.block
-; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm3
-; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm4
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vpaddd %xmm4, %xmm5, %xmm4
; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -501,7 +501,7 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
; AVX512F-NEXT: # %bb.2: # %middle.block
; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -530,7 +530,7 @@ define i32 @_Z10test_shortPsS_i_1024(ptr nocapture readonly, ptr nocapture reado
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -952,7 +952,7 @@ define i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocapture readonly
; AVX512-NEXT: jne .LBB6_1
; AVX512-NEXT: # %bb.2: # %middle.block
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -994,10 +994,10 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
; SSE2-NEXT: movl %edx, %eax
; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: xorl %ecx, %ecx
-; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: pxor %xmm4, %xmm4
-; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB7_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
@@ -1010,36 +1010,36 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm8[0],xmm10[1],xmm8[1],xmm10[2],xmm8[2],xmm10[3],xmm8[3],xmm10[4],xmm8[4],xmm10[5],xmm8[5],xmm10[6],xmm8[6],xmm10[7],xmm8[7]
; SSE2-NEXT: psraw $8, %xmm10
; SSE2-NEXT: pmaddwd %xmm9, %xmm10
-; SSE2-NEXT: paddd %xmm10, %xmm2
+; SSE2-NEXT: paddd %xmm10, %xmm4
; SSE2-NEXT: punpckhbw {{.*#+}} xmm7 = xmm7[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; SSE2-NEXT: psraw $8, %xmm7
; SSE2-NEXT: punpckhbw {{.*#+}} xmm8 = xmm8[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; SSE2-NEXT: psraw $8, %xmm8
; SSE2-NEXT: pmaddwd %xmm7, %xmm8
-; SSE2-NEXT: paddd %xmm8, %xmm4
+; SSE2-NEXT: paddd %xmm8, %xmm3
; SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm6[0],xmm7[1],xmm6[1],xmm7[2],xmm6[2],xmm7[3],xmm6[3],xmm7[4],xmm6[4],xmm7[5],xmm6[5],xmm7[6],xmm6[6],xmm7[7],xmm6[7]
; SSE2-NEXT: psraw $8, %xmm7
; SSE2-NEXT: punpcklbw {{.*#+}} xmm8 = xmm8[0],xmm5[0],xmm8[1],xmm5[1],xmm8[2],xmm5[2],xmm8[3],xmm5[3],xmm8[4],xmm5[4],xmm8[5],xmm5[5],xmm8[6],xmm5[6],xmm8[7],xmm5[7]
; SSE2-NEXT: psraw $8, %xmm8
; SSE2-NEXT: pmaddwd %xmm7, %xmm8
-; SSE2-NEXT: paddd %xmm8, %xmm1
+; SSE2-NEXT: paddd %xmm8, %xmm2
; SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; SSE2-NEXT: psraw $8, %xmm6
; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; SSE2-NEXT: psraw $8, %xmm5
; SSE2-NEXT: pmaddwd %xmm6, %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm3
+; SSE2-NEXT: paddd %xmm5, %xmm1
; SSE2-NEXT: addq $32, %rcx
; SSE2-NEXT: cmpq %rcx, %rax
; SSE2-NEXT: jne .LBB7_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm0, %xmm4
-; SSE2-NEXT: paddd %xmm0, %xmm3
-; SSE2-NEXT: paddd %xmm4, %xmm3
; SSE2-NEXT: paddd %xmm0, %xmm2
+; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: paddd %xmm0, %xmm3
; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: paddd %xmm2, %xmm1
; SSE2-NEXT: paddd %xmm3, %xmm1
+; SSE2-NEXT: paddd %xmm2, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; SSE2-NEXT: paddd %xmm1, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1052,10 +1052,10 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
; SSE42-NEXT: movl %edx, %eax
; SSE42-NEXT: pxor %xmm0, %xmm0
; SSE42-NEXT: xorl %ecx, %ecx
-; SSE42-NEXT: pxor %xmm2, %xmm2
; SSE42-NEXT: pxor %xmm4, %xmm4
-; SSE42-NEXT: pxor %xmm1, %xmm1
; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: pxor %xmm1, %xmm1
; SSE42-NEXT: .p2align 4
; SSE42-NEXT: .LBB7_1: # %vector.body
; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
@@ -1065,27 +1065,27 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
; SSE42-NEXT: pmovsxbw 24(%rdi,%rcx), %xmm8
; SSE42-NEXT: pmovsxbw 16(%rsi,%rcx), %xmm9
; SSE42-NEXT: pmaddwd %xmm5, %xmm9
-; SSE42-NEXT: paddd %xmm9, %xmm1
+; SSE42-NEXT: paddd %xmm9, %xmm2
; SSE42-NEXT: pmovsxbw 8(%rsi,%rcx), %xmm5
; SSE42-NEXT: pmaddwd %xmm6, %xmm5
-; SSE42-NEXT: paddd %xmm5, %xmm4
+; SSE42-NEXT: paddd %xmm5, %xmm3
; SSE42-NEXT: pmovsxbw (%rsi,%rcx), %xmm5
; SSE42-NEXT: pmaddwd %xmm7, %xmm5
-; SSE42-NEXT: paddd %xmm5, %xmm2
+; SSE42-NEXT: paddd %xmm5, %xmm4
; SSE42-NEXT: pmovsxbw 24(%rsi,%rcx), %xmm5
; SSE42-NEXT: pmaddwd %xmm8, %xmm5
-; SSE42-NEXT: paddd %xmm5, %xmm3
+; SSE42-NEXT: paddd %xmm5, %xmm1
; SSE42-NEXT: addq $32, %rcx
; SSE42-NEXT: cmpq %rcx, %rax
; SSE42-NEXT: jne .LBB7_1
; SSE42-NEXT: # %bb.2: # %middle.block
; SSE42-NEXT: paddd %xmm0, %xmm4
-; SSE42-NEXT: paddd %xmm0, %xmm3
-; SSE42-NEXT: paddd %xmm4, %xmm3
; SSE42-NEXT: paddd %xmm0, %xmm2
+; SSE42-NEXT: paddd %xmm4, %xmm2
+; SSE42-NEXT: paddd %xmm0, %xmm3
; SSE42-NEXT: paddd %xmm0, %xmm1
-; SSE42-NEXT: paddd %xmm2, %xmm1
; SSE42-NEXT: paddd %xmm3, %xmm1
+; SSE42-NEXT: paddd %xmm2, %xmm1
; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; SSE42-NEXT: paddd %xmm1, %xmm0
; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1127,16 +1127,16 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
; AVX1-NEXT: cmpq %rcx, %rax
; AVX1-NEXT: jne .LBB7_1
; AVX1-NEXT: # %bb.2: # %middle.block
-; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm3
-; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm4
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vpaddd %xmm4, %xmm5, %xmm4
; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1203,7 +1203,7 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
; AVX512F-NEXT: # %bb.2: # %middle.block
; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1233,7 +1233,7 @@ define i32 @_Z9test_charPcS_i_1024(ptr nocapture readonly, ptr nocapture readonl
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1693,7 +1693,7 @@ define i32 @test_unsigned_short_512(ptr nocapture readonly, ptr nocapture readon
; AVX512-NEXT: jne .LBB10_1
; AVX512-NEXT: # %bb.2: # %middle.block
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1733,15 +1733,15 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
; SSE2-LABEL: test_unsigned_short_1024:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: movl %edx, %eax
-; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: xorl %ecx, %ecx
; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pxor %xmm7, %xmm7
+; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB11_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
@@ -1752,27 +1752,27 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
; SSE2-NEXT: pmullw %xmm8, %xmm9
; SSE2-NEXT: movdqa %xmm9, %xmm8
; SSE2-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm10[4],xmm8[5],xmm10[5],xmm8[6],xmm10[6],xmm8[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm8, %xmm7
+; SSE2-NEXT: paddd %xmm8, %xmm5
; SSE2-NEXT: movdqu 32(%rdi,%rcx,2), %xmm8
; SSE2-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm10[0],xmm9[1],xmm10[1],xmm9[2],xmm10[2],xmm9[3],xmm10[3]
; SSE2-NEXT: movdqu 32(%rsi,%rcx,2), %xmm10
-; SSE2-NEXT: paddd %xmm9, %xmm5
+; SSE2-NEXT: paddd %xmm9, %xmm7
; SSE2-NEXT: movdqa %xmm10, %xmm9
; SSE2-NEXT: pmulhuw %xmm8, %xmm9
; SSE2-NEXT: pmullw %xmm8, %xmm10
; SSE2-NEXT: movdqa %xmm10, %xmm8
; SSE2-NEXT: punpckhwd {{.*#+}} xmm8 = xmm8[4],xmm9[4],xmm8[5],xmm9[5],xmm8[6],xmm9[6],xmm8[7],xmm9[7]
-; SSE2-NEXT: paddd %xmm8, %xmm6
+; SSE2-NEXT: paddd %xmm8, %xmm4
; SSE2-NEXT: movdqu (%rdi,%rcx,2), %xmm8
; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm9[0],xmm10[1],xmm9[1],xmm10[2],xmm9[2],xmm10[3],xmm9[3]
; SSE2-NEXT: movdqu (%rsi,%rcx,2), %xmm9
-; SSE2-NEXT: paddd %xmm10, %xmm4
+; SSE2-NEXT: paddd %xmm10, %xmm6
; SSE2-NEXT: movdqa %xmm9, %xmm10
; SSE2-NEXT: pmulhuw %xmm8, %xmm10
; SSE2-NEXT: pmullw %xmm8, %xmm9
; SSE2-NEXT: movdqa %xmm9, %xmm8
; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm10[0],xmm8[1],xmm10[1],xmm8[2],xmm10[2],xmm8[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm8, %xmm0
+; SSE2-NEXT: paddd %xmm8, %xmm2
; SSE2-NEXT: movdqu 16(%rdi,%rcx,2), %xmm8
; SSE2-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm10[4],xmm9[5],xmm10[5],xmm9[6],xmm10[6],xmm9[7],xmm10[7]
; SSE2-NEXT: movdqu 16(%rsi,%rcx,2), %xmm10
@@ -1784,37 +1784,37 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
; SSE2-NEXT: punpcklwd {{.*#+}} xmm8 = xmm8[0],xmm9[0],xmm8[1],xmm9[1],xmm8[2],xmm9[2],xmm8[3],xmm9[3]
; SSE2-NEXT: paddd %xmm8, %xmm1
; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm9[4],xmm10[5],xmm9[5],xmm10[6],xmm9[6],xmm10[7],xmm9[7]
-; SSE2-NEXT: paddd %xmm10, %xmm2
+; SSE2-NEXT: paddd %xmm10, %xmm0
; SSE2-NEXT: addq $16, %rcx
; SSE2-NEXT: cmpq %rcx, %rax
; SSE2-NEXT: jne .LBB11_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm6, %xmm3
-; SSE2-NEXT: paddd %xmm7, %xmm2
-; SSE2-NEXT: paddd %xmm3, %xmm2
-; SSE2-NEXT: paddd %xmm4, %xmm0
-; SSE2-NEXT: paddd %xmm5, %xmm1
-; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: paddd %xmm6, %xmm2
+; SSE2-NEXT: paddd %xmm7, %xmm1
; SSE2-NEXT: paddd %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: paddd %xmm4, %xmm3
+; SSE2-NEXT: paddd %xmm5, %xmm0
+; SSE2-NEXT: paddd %xmm3, %xmm0
; SSE2-NEXT: paddd %xmm1, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: retq
;
; SSE42-LABEL: test_unsigned_short_1024:
; SSE42: # %bb.0: # %entry
; SSE42-NEXT: movl %edx, %eax
-; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: pxor %xmm2, %xmm2
; SSE42-NEXT: xorl %ecx, %ecx
; SSE42-NEXT: pxor %xmm3, %xmm3
; SSE42-NEXT: pxor %xmm1, %xmm1
-; SSE42-NEXT: pxor %xmm2, %xmm2
-; SSE42-NEXT: pxor %xmm4, %xmm4
+; SSE42-NEXT: pxor %xmm0, %xmm0
; SSE42-NEXT: pxor %xmm6, %xmm6
-; SSE42-NEXT: pxor %xmm5, %xmm5
+; SSE42-NEXT: pxor %xmm4, %xmm4
; SSE42-NEXT: pxor %xmm7, %xmm7
+; SSE42-NEXT: pxor %xmm5, %xmm5
; SSE42-NEXT: .p2align 4
; SSE42-NEXT: .LBB11_1: # %vector.body
; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
@@ -1830,15 +1830,11 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
; SSE42-NEXT: pmovzxwd {{.*#+}} xmm9 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; SSE42-NEXT: pmulld %xmm8, %xmm9
; SSE42-NEXT: pmovzxwd {{.*#+}} xmm8 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; SSE42-NEXT: paddd %xmm9, %xmm0
-; SSE42-NEXT: pmovzxwd {{.*#+}} xmm9 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; SSE42-NEXT: pmulld %xmm8, %xmm9
-; SSE42-NEXT: pmovzxwd {{.*#+}} xmm8 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; SSE42-NEXT: paddd %xmm9, %xmm2
; SSE42-NEXT: pmovzxwd {{.*#+}} xmm9 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; SSE42-NEXT: pmulld %xmm8, %xmm9
; SSE42-NEXT: pmovzxwd {{.*#+}} xmm8 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; SSE42-NEXT: paddd %xmm9, %xmm4
+; SSE42-NEXT: paddd %xmm9, %xmm0
; SSE42-NEXT: pmovzxwd {{.*#+}} xmm9 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; SSE42-NEXT: pmulld %xmm8, %xmm9
; SSE42-NEXT: pmovzxwd {{.*#+}} xmm8 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
@@ -1846,26 +1842,30 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
; SSE42-NEXT: pmovzxwd {{.*#+}} xmm9 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; SSE42-NEXT: pmulld %xmm8, %xmm9
; SSE42-NEXT: pmovzxwd {{.*#+}} xmm8 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
-; SSE42-NEXT: paddd %xmm9, %xmm5
+; SSE42-NEXT: paddd %xmm9, %xmm4
; SSE42-NEXT: pmovzxwd {{.*#+}} xmm9 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; SSE42-NEXT: pmulld %xmm8, %xmm9
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm8 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
; SSE42-NEXT: paddd %xmm9, %xmm7
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm9 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; SSE42-NEXT: pmulld %xmm8, %xmm9
+; SSE42-NEXT: paddd %xmm9, %xmm5
; SSE42-NEXT: addq $16, %rcx
; SSE42-NEXT: cmpq %rcx, %rax
; SSE42-NEXT: jne .LBB11_1
; SSE42-NEXT: # %bb.2: # %middle.block
-; SSE42-NEXT: paddd %xmm6, %xmm3
-; SSE42-NEXT: paddd %xmm7, %xmm2
-; SSE42-NEXT: paddd %xmm3, %xmm2
-; SSE42-NEXT: paddd %xmm4, %xmm0
-; SSE42-NEXT: paddd %xmm5, %xmm1
-; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: paddd %xmm6, %xmm2
+; SSE42-NEXT: paddd %xmm7, %xmm1
; SSE42-NEXT: paddd %xmm2, %xmm1
-; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE42-NEXT: paddd %xmm4, %xmm3
+; SSE42-NEXT: paddd %xmm5, %xmm0
+; SSE42-NEXT: paddd %xmm3, %xmm0
; SSE42-NEXT: paddd %xmm1, %xmm0
-; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE42-NEXT: paddd %xmm0, %xmm1
-; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: movd %xmm0, %eax
; SSE42-NEXT: retq
;
; AVX1-LABEL: test_unsigned_short_1024:
@@ -1923,17 +1923,17 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
; AVX1-NEXT: cmpq %rcx, %rax
; AVX1-NEXT: jne .LBB11_1
; AVX1-NEXT: # %bb.2: # %middle.block
-; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm4
-; AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm5
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm5
+; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6
+; AVX1-NEXT: vpaddd %xmm6, %xmm5, %xmm5
; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
; AVX1-NEXT: vpaddd %xmm2, %xmm1, %xmm1
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpaddd %xmm0, %xmm4, %xmm0
+; AVX1-NEXT: vpaddd %xmm4, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -2009,7 +2009,7 @@ define i32 @test_unsigned_short_1024(ptr nocapture readonly, ptr nocapture reado
; AVX512-NEXT: # %bb.2: # %middle.block
; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -3545,7 +3545,7 @@ define i32 @add_used_by_loop_phi(ptr %a, ptr %b, i64 %offset_a, i64 %offset_b, i
; AVX512-NEXT: jb .LBB40_1
; AVX512-NEXT: # %bb.2: # %afterloop
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/min-legal-vector-width.ll b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
index 4c89fa95fe4b2..e1acb933ed2a1 100644
--- a/llvm/test/CodeGen/X86/min-legal-vector-width.ll
+++ b/llvm/test/CodeGen/X86/min-legal-vector-width.ll
@@ -320,7 +320,7 @@ define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocaptur
; CHECK-SKX-NEXT: # %bb.2: # %middle.block
; CHECK-SKX-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; CHECK-SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-SKX-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-SKX-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-SKX-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -350,7 +350,7 @@ define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocaptur
; CHECK-AVX512-NEXT: # %bb.2: # %middle.block
; CHECK-AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; CHECK-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -380,7 +380,7 @@ define dso_local i32 @_Z9test_charPcS_i_512(ptr nocapture readonly, ptr nocaptur
; CHECK-VBMI-NEXT: # %bb.2: # %middle.block
; CHECK-VBMI-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; CHECK-VBMI-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-VBMI-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-VBMI-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -537,7 +537,7 @@ define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {
; CHECK-SKX-NEXT: jne .LBB11_1
; CHECK-SKX-NEXT: # %bb.2: # %middle.block
; CHECK-SKX-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-SKX-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-SKX-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-SKX-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-SKX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-SKX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -562,7 +562,7 @@ define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {
; CHECK-AVX512-NEXT: jne .LBB11_1
; CHECK-AVX512-NEXT: # %bb.2: # %middle.block
; CHECK-AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -587,7 +587,7 @@ define dso_local i32 @sad_16i8_512() "min-legal-vector-width"="512" {
; CHECK-VBMI-NEXT: jne .LBB11_1
; CHECK-VBMI-NEXT: # %bb.2: # %middle.block
; CHECK-VBMI-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-VBMI-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; CHECK-VBMI-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; CHECK-VBMI-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-VBMI-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; CHECK-VBMI-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/phaddsub-extract.ll b/llvm/test/CodeGen/X86/phaddsub-extract.ll
index b38a10c7e4263..b9b07c22bb034 100644
--- a/llvm/test/CodeGen/X86/phaddsub-extract.ll
+++ b/llvm/test/CodeGen/X86/phaddsub-extract.ll
@@ -1699,7 +1699,8 @@ define i32 @partial_reduction_add_v8i32(<8 x i32> %x) {
;
; AVX-FAST-LABEL: partial_reduction_add_v8i32:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX-FAST-NEXT: vmovd %xmm0, %eax
; AVX-FAST-NEXT: vzeroupper
@@ -1740,13 +1741,33 @@ define i32 @partial_reduction_add_v16i32(<16 x i32> %x) {
; AVX-SLOW-NEXT: vzeroupper
; AVX-SLOW-NEXT: retq
;
-; AVX-FAST-LABEL: partial_reduction_add_v16i32:
-; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX-FAST-NEXT: vmovd %xmm0, %eax
-; AVX-FAST-NEXT: vzeroupper
-; AVX-FAST-NEXT: retq
+; AVX1-FAST-LABEL: partial_reduction_add_v16i32:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; AVX1-FAST-NEXT: vzeroupper
+; AVX1-FAST-NEXT: retq
+;
+; AVX2-FAST-LABEL: partial_reduction_add_v16i32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vmovd %xmm0, %eax
+; AVX2-FAST-NEXT: vzeroupper
+; AVX2-FAST-NEXT: retq
+;
+; AVX512-FAST-LABEL: partial_reduction_add_v16i32:
+; AVX512-FAST: # %bb.0:
+; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vmovd %xmm0, %eax
+; AVX512-FAST-NEXT: vzeroupper
+; AVX512-FAST-NEXT: retq
%x23 = shufflevector <16 x i32> %x, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%x0213 = add <16 x i32> %x, %x23
%x13 = shufflevector <16 x i32> %x0213, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -1937,8 +1958,10 @@ define i16 @hadd16_8(<8 x i16> %x223) {
;
; SSE3-FAST-LABEL: hadd16_8:
; SSE3-FAST: # %bb.0:
-; SSE3-FAST-NEXT: phaddw %xmm0, %xmm0
-; SSE3-FAST-NEXT: phaddw %xmm0, %xmm0
+; SSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE3-FAST-NEXT: paddw %xmm0, %xmm1
+; SSE3-FAST-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE3-FAST-NEXT: paddw %xmm1, %xmm0
; SSE3-FAST-NEXT: phaddw %xmm0, %xmm0
; SSE3-FAST-NEXT: movd %xmm0, %eax
; SSE3-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1958,8 +1981,10 @@ define i16 @hadd16_8(<8 x i16> %x223) {
;
; AVX-FAST-LABEL: hadd16_8:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
-; AVX-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX-FAST-NEXT: vmovd %xmm0, %eax
; AVX-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1986,9 +2011,10 @@ define i32 @hadd32_4(<4 x i32> %x225) {
;
; SSE3-FAST-LABEL: hadd32_4:
; SSE3-FAST: # %bb.0:
-; SSE3-FAST-NEXT: phaddd %xmm0, %xmm0
-; SSE3-FAST-NEXT: phaddd %xmm0, %xmm0
-; SSE3-FAST-NEXT: movd %xmm0, %eax
+; SSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE3-FAST-NEXT: paddd %xmm0, %xmm1
+; SSE3-FAST-NEXT: phaddd %xmm1, %xmm1
+; SSE3-FAST-NEXT: movd %xmm1, %eax
; SSE3-FAST-NEXT: retq
;
; AVX-SLOW-LABEL: hadd32_4:
@@ -2002,7 +2028,8 @@ define i32 @hadd32_4(<4 x i32> %x225) {
;
; AVX-FAST-LABEL: hadd32_4:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX-FAST-NEXT: vmovd %xmm0, %eax
; AVX-FAST-NEXT: retq
@@ -2044,7 +2071,8 @@ define i32 @hadd32_8(<8 x i32> %x225) {
;
; AVX-FAST-LABEL: hadd32_8:
; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX-FAST-NEXT: vmovd %xmm0, %eax
; AVX-FAST-NEXT: vzeroupper
@@ -2085,13 +2113,33 @@ define i32 @hadd32_16(<16 x i32> %x225) {
; AVX-SLOW-NEXT: vzeroupper
; AVX-SLOW-NEXT: retq
;
-; AVX-FAST-LABEL: hadd32_16:
-; AVX-FAST: # %bb.0:
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX-FAST-NEXT: vmovd %xmm0, %eax
-; AVX-FAST-NEXT: vzeroupper
-; AVX-FAST-NEXT: retq
+; AVX1-FAST-LABEL: hadd32_16:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; AVX1-FAST-NEXT: vzeroupper
+; AVX1-FAST-NEXT: retq
+;
+; AVX2-FAST-LABEL: hadd32_16:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vmovd %xmm0, %eax
+; AVX2-FAST-NEXT: vzeroupper
+; AVX2-FAST-NEXT: retq
+;
+; AVX512-FAST-LABEL: hadd32_16:
+; AVX512-FAST: # %bb.0:
+; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vmovd %xmm0, %eax
+; AVX512-FAST-NEXT: vzeroupper
+; AVX512-FAST-NEXT: retq
%x226 = shufflevector <16 x i32> %x225, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%x227 = add <16 x i32> %x225, %x226
%x228 = shufflevector <16 x i32> %x227, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
@@ -2103,8 +2151,10 @@ define i32 @hadd32_16(<16 x i32> %x225) {
define i16 @hadd16_8_optsize(<8 x i16> %x223) optsize {
; SSE3-LABEL: hadd16_8_optsize:
; SSE3: # %bb.0:
-; SSE3-NEXT: phaddw %xmm0, %xmm0
-; SSE3-NEXT: phaddw %xmm0, %xmm0
+; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE3-NEXT: paddw %xmm0, %xmm1
+; SSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE3-NEXT: paddw %xmm1, %xmm0
; SSE3-NEXT: phaddw %xmm0, %xmm0
; SSE3-NEXT: movd %xmm0, %eax
; SSE3-NEXT: # kill: def $ax killed $ax killed $eax
@@ -2112,8 +2162,10 @@ define i16 @hadd16_8_optsize(<8 x i16> %x223) optsize {
;
; AVX-LABEL: hadd16_8_optsize:
; AVX: # %bb.0:
-; AVX-NEXT: vphaddw %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX-NEXT: vmovd %xmm0, %eax
; AVX-NEXT: # kill: def $ax killed $ax killed $eax
@@ -2131,14 +2183,16 @@ define i16 @hadd16_8_optsize(<8 x i16> %x223) optsize {
define i32 @hadd32_4_optsize(<4 x i32> %x225) optsize {
; SSE3-LABEL: hadd32_4_optsize:
; SSE3: # %bb.0:
-; SSE3-NEXT: phaddd %xmm0, %xmm0
-; SSE3-NEXT: phaddd %xmm0, %xmm0
-; SSE3-NEXT: movd %xmm0, %eax
+; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE3-NEXT: paddd %xmm0, %xmm1
+; SSE3-NEXT: phaddd %xmm1, %xmm1
+; SSE3-NEXT: movd %xmm1, %eax
; SSE3-NEXT: retq
;
; AVX-LABEL: hadd32_4_optsize:
; AVX: # %bb.0:
-; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX-NEXT: vmovd %xmm0, %eax
; AVX-NEXT: retq
@@ -2153,14 +2207,16 @@ define i32 @hadd32_4_optsize(<4 x i32> %x225) optsize {
define i32 @hadd32_4_pgso(<4 x i32> %x225) !prof !14 {
; SSE3-LABEL: hadd32_4_pgso:
; SSE3: # %bb.0:
-; SSE3-NEXT: phaddd %xmm0, %xmm0
-; SSE3-NEXT: phaddd %xmm0, %xmm0
-; SSE3-NEXT: movd %xmm0, %eax
+; SSE3-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE3-NEXT: paddd %xmm0, %xmm1
+; SSE3-NEXT: phaddd %xmm1, %xmm1
+; SSE3-NEXT: movd %xmm1, %eax
; SSE3-NEXT: retq
;
; AVX-LABEL: hadd32_4_pgso:
; AVX: # %bb.0:
-; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX-NEXT: vmovd %xmm0, %eax
; AVX-NEXT: retq
@@ -2183,7 +2239,8 @@ define i32 @hadd32_8_optsize(<8 x i32> %x225) optsize {
;
; AVX-LABEL: hadd32_8_optsize:
; AVX: # %bb.0:
-; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX-NEXT: vmovd %xmm0, %eax
; AVX-NEXT: vzeroupper
@@ -2205,13 +2262,61 @@ define i32 @hadd32_16_optsize(<16 x i32> %x225) optsize {
; SSE3-NEXT: movd %xmm1, %eax
; SSE3-NEXT: retq
;
-; AVX-LABEL: hadd32_16_optsize:
-; AVX: # %bb.0:
-; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: vzeroupper
-; AVX-NEXT: retq
+; AVX1-SLOW-LABEL: hadd32_16_optsize:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT: vzeroupper
+; AVX1-SLOW-NEXT: retq
+;
+; AVX1-FAST-LABEL: hadd32_16_optsize:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; AVX1-FAST-NEXT: vzeroupper
+; AVX1-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: hadd32_16_optsize:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vmovd %xmm0, %eax
+; AVX2-SLOW-NEXT: vzeroupper
+; AVX2-SLOW-NEXT: retq
+;
+; AVX2-FAST-LABEL: hadd32_16_optsize:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vmovd %xmm0, %eax
+; AVX2-FAST-NEXT: vzeroupper
+; AVX2-FAST-NEXT: retq
+;
+; AVX512-SLOW-LABEL: hadd32_16_optsize:
+; AVX512-SLOW: # %bb.0:
+; AVX512-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-SLOW-NEXT: vmovd %xmm0, %eax
+; AVX512-SLOW-NEXT: vzeroupper
+; AVX512-SLOW-NEXT: retq
+;
+; AVX512-FAST-LABEL: hadd32_16_optsize:
+; AVX512-FAST: # %bb.0:
+; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-FAST-NEXT: vmovd %xmm0, %eax
+; AVX512-FAST-NEXT: vzeroupper
+; AVX512-FAST-NEXT: retq
%x226 = shufflevector <16 x i32> %x225, <16 x i32> undef, <16 x i32> <i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
%x227 = add <16 x i32> %x225, %x226
%x228 = shufflevector <16 x i32> %x227, <16 x i32> undef, <16 x i32> <i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
diff --git a/llvm/test/CodeGen/X86/phaddsub.ll b/llvm/test/CodeGen/X86/phaddsub.ll
index 1ec4738220b6c..80d06eb631340 100644
--- a/llvm/test/CodeGen/X86/phaddsub.ll
+++ b/llvm/test/CodeGen/X86/phaddsub.ll
@@ -625,10 +625,10 @@ define i32 @PR39936_v8i32(<8 x i32>) {
;
; SSSE3-FAST-LABEL: PR39936_v8i32:
; SSSE3-FAST: # %bb.0:
-; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm1
-; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm1
-; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm1
-; SSSE3-FAST-NEXT: movd %xmm1, %eax
+; SSSE3-FAST-NEXT: paddd %xmm1, %xmm0
+; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm0
+; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm0
+; SSSE3-FAST-NEXT: movd %xmm0, %eax
; SSSE3-FAST-NEXT: retq
;
; AVX1-SLOW-LABEL: PR39936_v8i32:
@@ -646,7 +646,7 @@ define i32 @PR39936_v8i32(<8 x i32>) {
; AVX1-FAST-LABEL: PR39936_v8i32:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
@@ -668,7 +668,7 @@ define i32 @PR39936_v8i32(<8 x i32>) {
; AVX2-FAST-LABEL: PR39936_v8i32:
; AVX2-FAST: # %bb.0:
; AVX2-FAST-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
+; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX2-FAST-NEXT: vmovd %xmm0, %eax
diff --git a/llvm/test/CodeGen/X86/pr173924.ll b/llvm/test/CodeGen/X86/pr173924.ll
index d86b82edb7249..5cbd123371b35 100644
--- a/llvm/test/CodeGen/X86/pr173924.ll
+++ b/llvm/test/CodeGen/X86/pr173924.ll
@@ -6,30 +6,40 @@ define i256 @PR173924(<8 x i256> %a0) {
; CHECK-LABEL: PR173924:
; CHECK: # %bb.0:
; CHECK-NEXT: movq %rdi, %rax
-; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %ecx
-; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %edi
-; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %r8d
-; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %edx
-; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %r10d
-; CHECK-NEXT: movl {{[0-9]+}}(%rsp), %r11d
-; CHECK-NEXT: andl $1, %r11d
-; CHECK-NEXT: andl $1, %r9d
-; CHECK-NEXT: addl %r11d, %r9d
-; CHECK-NEXT: andl $1, %r10d
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r10
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r8
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %r11
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdi
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rdx
+; CHECK-NEXT: movq {{[0-9]+}}(%rsp), %rcx
+; CHECK-NEXT: andl $1, %ecx
; CHECK-NEXT: andl $1, %edx
-; CHECK-NEXT: addl %r10d, %edx
-; CHECK-NEXT: addl %r9d, %edx
+; CHECK-NEXT: andl $1, %edi
+; CHECK-NEXT: andl $1, %r11d
; CHECK-NEXT: andl $1, %r8d
+; CHECK-NEXT: addq %r11, %r8
+; CHECK-NEXT: andl $1, %r10d
+; CHECK-NEXT: andl $1, %r9d
; CHECK-NEXT: andl $1, %esi
-; CHECK-NEXT: addl %r8d, %esi
-; CHECK-NEXT: andl $1, %edi
-; CHECK-NEXT: andl $1, %ecx
-; CHECK-NEXT: addl %edi, %ecx
-; CHECK-NEXT: addl %esi, %ecx
-; CHECK-NEXT: addl %edx, %ecx
-; CHECK-NEXT: vmovd %ecx, %xmm0
-; CHECK-NEXT: vmovdqu %ymm0, (%rax)
-; CHECK-NEXT: vzeroupper
+; CHECK-NEXT: addq %r9, %rsi
+; CHECK-NEXT: addq %r10, %rsi
+; CHECK-NEXT: addq %r8, %rsi
+; CHECK-NEXT: xorl %r8d, %r8d
+; CHECK-NEXT: addq %rdi, %rsi
+; CHECK-NEXT: setb %r8b
+; CHECK-NEXT: addq %rdx, %rsi
+; CHECK-NEXT: adcq $0, %r8
+; CHECK-NEXT: setb %dl
+; CHECK-NEXT: movzbl %dl, %edx
+; CHECK-NEXT: addq %rcx, %rsi
+; CHECK-NEXT: adcq $0, %r8
+; CHECK-NEXT: adcq $0, %rdx
+; CHECK-NEXT: setb %cl
+; CHECK-NEXT: movzbl %cl, %ecx
+; CHECK-NEXT: movq %rsi, (%rax)
+; CHECK-NEXT: movq %r8, 8(%rax)
+; CHECK-NEXT: movq %rdx, 16(%rax)
+; CHECK-NEXT: movq %rcx, 24(%rax)
; CHECK-NEXT: retq
%m = and <8 x i256> %a0, splat (i256 1)
%r = call i256 @llvm.vector.reduce.add.v8i256(<8 x i256> %m)
diff --git a/llvm/test/CodeGen/X86/pr62286.ll b/llvm/test/CodeGen/X86/pr62286.ll
index 161e9651a9cf2..222a20936d375 100644
--- a/llvm/test/CodeGen/X86/pr62286.ll
+++ b/llvm/test/CodeGen/X86/pr62286.ll
@@ -8,17 +8,18 @@ define i64 @PR62286(i32 %a) {
; SSE-LABEL: PR62286:
; SSE: # %bb.0:
; SSE-NEXT: movd %edi, %xmm0
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,1,1,0]
-; SSE-NEXT: paddd %xmm1, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,0,1,0]
+; SSE-NEXT: paddd %xmm0, %xmm0
; SSE-NEXT: pxor %xmm2, %xmm2
-; SSE-NEXT: pxor %xmm3, %xmm3
-; SSE-NEXT: pcmpgtd %xmm1, %xmm3
-; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,0,1,0]
-; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; SSE-NEXT: pcmpgtd %xmm0, %xmm2
; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT: pxor %xmm3, %xmm3
+; SSE-NEXT: pcmpgtd %xmm0, %xmm3
+; SSE-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm3[0],xmm0[1],xmm3[1]
+; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE-NEXT: pcmpgtd %xmm1, %xmm2
+; SSE-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE-NEXT: paddq %xmm0, %xmm1
+; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; SSE-NEXT: paddq %xmm1, %xmm0
; SSE-NEXT: movq %xmm0, %rax
; SSE-NEXT: retq
@@ -73,7 +74,7 @@ define i64 @PR62286(i32 %a) {
; AVX512-NEXT: vmovdqa32 %zmm1, %zmm0 {%k1}
; AVX512-NEXT: vpmovsxdq %ymm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/sad.ll b/llvm/test/CodeGen/X86/sad.ll
index 095c8cfa1717c..7a31c896cce9f 100644
--- a/llvm/test/CodeGen/X86/sad.ll
+++ b/llvm/test/CodeGen/X86/sad.ll
@@ -101,7 +101,7 @@ define dso_local i32 @sad_16i8() nounwind {
; AVX512-NEXT: jne .LBB0_1
; AVX512-NEXT: # %bb.2: # %middle.block
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -158,8 +158,8 @@ define dso_local i32 @sad_32i8() nounwind {
; SSE2-NEXT: addq $32, %rax
; SSE2-NEXT: jne .LBB1_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm0, %xmm2
; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm2
; SSE2-NEXT: paddd %xmm0, %xmm0
; SSE2-NEXT: paddd %xmm0, %xmm1
; SSE2-NEXT: paddd %xmm2, %xmm0
@@ -190,15 +190,15 @@ define dso_local i32 @sad_32i8() nounwind {
; AVX1-NEXT: addq $32, %rax
; AVX1-NEXT: jne .LBB1_1
; AVX1-NEXT: # %bb.2: # %middle.block
-; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT: vpaddd %xmm4, %xmm4, %xmm5
-; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
+; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
+; AVX1-NEXT: vpaddd %xmm3, %xmm3, %xmm4
; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vpaddd %xmm3, %xmm2, %xmm2
+; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm3
; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
+; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
+; AVX1-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -250,7 +250,7 @@ define dso_local i32 @sad_32i8() nounwind {
; AVX512-NEXT: # %bb.2: # %middle.block
; AVX512-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -291,18 +291,18 @@ middle.block:
define dso_local i32 @sad_avx64i8() nounwind {
; SSE2-LABEL: sad_avx64i8:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
-; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB2_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
; SSE2-NEXT: movdqa a+1024(%rax), %xmm5
; SSE2-NEXT: psadbw b+1024(%rax), %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm0
+; SSE2-NEXT: paddd %xmm5, %xmm4
; SSE2-NEXT: movdqa a+1040(%rax), %xmm5
; SSE2-NEXT: psadbw b+1040(%rax), %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm3
@@ -311,28 +311,28 @@ define dso_local i32 @sad_avx64i8() nounwind {
; SSE2-NEXT: paddd %xmm5, %xmm2
; SSE2-NEXT: movdqa a+1072(%rax), %xmm5
; SSE2-NEXT: psadbw b+1072(%rax), %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm1
+; SSE2-NEXT: paddd %xmm5, %xmm0
; SSE2-NEXT: addq $64, %rax
; SSE2-NEXT: jne .LBB2_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: paddd %xmm1, %xmm4
; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm5
-; SSE2-NEXT: paddd %xmm4, %xmm0
-; SSE2-NEXT: paddd %xmm4, %xmm1
-; SSE2-NEXT: paddd %xmm4, %xmm3
+; SSE2-NEXT: paddd %xmm5, %xmm4
+; SSE2-NEXT: paddd %xmm1, %xmm2
+; SSE2-NEXT: paddd %xmm5, %xmm2
+; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: paddd %xmm1, %xmm3
; SSE2-NEXT: paddd %xmm5, %xmm3
-; SSE2-NEXT: paddd %xmm5, %xmm1
-; SSE2-NEXT: paddd %xmm3, %xmm1
-; SSE2-NEXT: paddd %xmm5, %xmm0
-; SSE2-NEXT: paddd %xmm2, %xmm5
-; SSE2-NEXT: paddd %xmm0, %xmm5
-; SSE2-NEXT: paddd %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; SSE2-NEXT: paddd %xmm1, %xmm0
; SSE2-NEXT: paddd %xmm5, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: retq
;
; AVX1-LABEL: sad_avx64i8:
@@ -363,22 +363,22 @@ define dso_local i32 @sad_avx64i8() nounwind {
; AVX1-NEXT: addq $64, %rax
; AVX1-NEXT: jne .LBB2_1
; AVX1-NEXT: # %bb.2: # %middle.block
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
; AVX1-NEXT: vpaddd %xmm4, %xmm4, %xmm5
-; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6
-; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm7
-; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm8
-; AVX1-NEXT: vpaddd %xmm0, %xmm8, %xmm8
-; AVX1-NEXT: vpaddd %xmm2, %xmm8, %xmm2
-; AVX1-NEXT: vpaddd %xmm7, %xmm0, %xmm0
+; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm0, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm6, %xmm2
-; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -391,26 +391,26 @@ define dso_local i32 @sad_avx64i8() nounwind {
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB2_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
; AVX2-NEXT: vmovdqa a+1024(%rax), %ymm3
; AVX2-NEXT: vpsadbw b+1024(%rax), %ymm3, %ymm3
-; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vmovdqa a+1056(%rax), %ymm3
; AVX2-NEXT: vpsadbw b+1056(%rax), %ymm3, %ymm3
-; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
; AVX2-NEXT: addq $64, %rax
; AVX2-NEXT: jne .LBB2_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm2
; AVX2-NEXT: vpaddd %ymm0, %ymm0, %ymm3
+; AVX2-NEXT: vpaddd %ymm3, %ymm2, %ymm2
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vpaddd %ymm3, %ymm0, %ymm0
-; AVX2-NEXT: vpaddd %ymm3, %ymm2, %ymm1
-; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -442,7 +442,7 @@ define dso_local i32 @sad_avx64i8() nounwind {
; AVX512F-NEXT: vpaddd %zmm0, %zmm0, %zmm0
; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -471,7 +471,7 @@ define dso_local i32 @sad_avx64i8() nounwind {
; AVX512BW-NEXT: vpaddd %zmm0, %zmm0, %zmm0
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -513,18 +513,18 @@ middle.block:
define i32 @sad_avx64i8_prefer_256() nounwind "min-legal-vector-width"="256" "prefer-vector-width"="256" {
; SSE2-LABEL: sad_avx64i8_prefer_256:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
-; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB3_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
; SSE2-NEXT: movdqa a+1024(%rax), %xmm5
; SSE2-NEXT: psadbw b+1024(%rax), %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm0
+; SSE2-NEXT: paddd %xmm5, %xmm4
; SSE2-NEXT: movdqa a+1040(%rax), %xmm5
; SSE2-NEXT: psadbw b+1040(%rax), %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm3
@@ -533,28 +533,28 @@ define i32 @sad_avx64i8_prefer_256() nounwind "min-legal-vector-width"="256" "pr
; SSE2-NEXT: paddd %xmm5, %xmm2
; SSE2-NEXT: movdqa a+1072(%rax), %xmm5
; SSE2-NEXT: psadbw b+1072(%rax), %xmm5
-; SSE2-NEXT: paddd %xmm5, %xmm1
+; SSE2-NEXT: paddd %xmm5, %xmm0
; SSE2-NEXT: addq $64, %rax
; SSE2-NEXT: jne .LBB3_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: paddd %xmm1, %xmm4
; SSE2-NEXT: pxor %xmm5, %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm5
-; SSE2-NEXT: paddd %xmm4, %xmm0
-; SSE2-NEXT: paddd %xmm4, %xmm1
-; SSE2-NEXT: paddd %xmm4, %xmm3
+; SSE2-NEXT: paddd %xmm5, %xmm4
+; SSE2-NEXT: paddd %xmm1, %xmm2
+; SSE2-NEXT: paddd %xmm5, %xmm2
+; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: paddd %xmm1, %xmm3
; SSE2-NEXT: paddd %xmm5, %xmm3
-; SSE2-NEXT: paddd %xmm5, %xmm1
-; SSE2-NEXT: paddd %xmm3, %xmm1
-; SSE2-NEXT: paddd %xmm5, %xmm0
-; SSE2-NEXT: paddd %xmm2, %xmm5
-; SSE2-NEXT: paddd %xmm0, %xmm5
-; SSE2-NEXT: paddd %xmm1, %xmm5
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; SSE2-NEXT: paddd %xmm1, %xmm0
; SSE2-NEXT: paddd %xmm5, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: retq
;
; AVX1-LABEL: sad_avx64i8_prefer_256:
@@ -585,22 +585,22 @@ define i32 @sad_avx64i8_prefer_256() nounwind "min-legal-vector-width"="256" "pr
; AVX1-NEXT: addq $64, %rax
; AVX1-NEXT: jne .LBB3_1
; AVX1-NEXT: # %bb.2: # %middle.block
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
; AVX1-NEXT: vpaddd %xmm4, %xmm4, %xmm5
-; AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6
-; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm7
-; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm8
-; AVX1-NEXT: vpaddd %xmm0, %xmm8, %xmm8
-; AVX1-NEXT: vpaddd %xmm2, %xmm8, %xmm2
-; AVX1-NEXT: vpaddd %xmm7, %xmm0, %xmm0
+; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; AVX1-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm2, %xmm2
+; AVX1-NEXT: vpaddd %xmm0, %xmm0, %xmm4
+; AVX1-NEXT: vpaddd %xmm4, %xmm0, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX1-NEXT: vpaddd %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: vpaddd %xmm5, %xmm4, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm6, %xmm2
-; AVX1-NEXT: vpaddd %xmm1, %xmm3, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm2, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -613,26 +613,26 @@ define i32 @sad_avx64i8_prefer_256() nounwind "min-legal-vector-width"="256" "pr
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB3_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
; AVX2-NEXT: vmovdqa a+1024(%rax), %ymm3
; AVX2-NEXT: vpsadbw b+1024(%rax), %ymm3, %ymm3
-; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vmovdqa a+1056(%rax), %ymm3
; AVX2-NEXT: vpsadbw b+1056(%rax), %ymm3, %ymm3
-; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
; AVX2-NEXT: addq $64, %rax
; AVX2-NEXT: jne .LBB3_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm2
; AVX2-NEXT: vpaddd %ymm0, %ymm0, %ymm3
+; AVX2-NEXT: vpaddd %ymm3, %ymm2, %ymm2
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vpaddd %ymm3, %ymm0, %ymm0
-; AVX2-NEXT: vpaddd %ymm3, %ymm2, %ymm1
-; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -664,7 +664,7 @@ define i32 @sad_avx64i8_prefer_256() nounwind "min-legal-vector-width"="256" "pr
; AVX512F-NEXT: vpaddd %zmm0, %zmm0, %zmm0
; AVX512F-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512F-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -679,26 +679,26 @@ define i32 @sad_avx64i8_prefer_256() nounwind "min-legal-vector-width"="256" "pr
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB3_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
; AVX512BW-NEXT: vmovdqa a+1024(%rax), %ymm3
; AVX512BW-NEXT: vpsadbw b+1024(%rax), %ymm3, %ymm3
-; AVX512BW-NEXT: vpaddd %ymm1, %ymm3, %ymm1
+; AVX512BW-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; AVX512BW-NEXT: vmovdqa a+1056(%rax), %ymm3
; AVX512BW-NEXT: vpsadbw b+1056(%rax), %ymm3, %ymm3
-; AVX512BW-NEXT: vpaddd %ymm2, %ymm3, %ymm2
+; AVX512BW-NEXT: vpaddd %ymm1, %ymm3, %ymm1
; AVX512BW-NEXT: addq $64, %rax
; AVX512BW-NEXT: jne .LBB3_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vpaddd %ymm0, %ymm2, %ymm2
; AVX512BW-NEXT: vpaddd %ymm0, %ymm0, %ymm3
+; AVX512BW-NEXT: vpaddd %ymm3, %ymm2, %ymm2
; AVX512BW-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX512BW-NEXT: vpaddd %ymm3, %ymm0, %ymm0
-; AVX512BW-NEXT: vpaddd %ymm3, %ymm2, %ymm1
-; AVX512BW-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT: vpaddd %ymm0, %ymm2, %ymm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1162,10 +1162,12 @@ define dso_local i32 @sad_unroll_nonzero_initial(ptr %arg, ptr %arg1, ptr %arg2,
; AVX2-NEXT: vmovdqu (%rdx), %xmm1
; AVX2-NEXT: vpsadbw (%rcx), %xmm1, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [1,0,0,0]
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: incl %eax
; AVX2-NEXT: retq
;
; AVX512-LABEL: sad_unroll_nonzero_initial:
@@ -1176,11 +1178,9 @@ define dso_local i32 @sad_unroll_nonzero_initial(ptr %arg, ptr %arg1, ptr %arg2,
; AVX512-NEXT: vpsadbw (%rcx), %xmm1, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddd {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [1,0,0,0]
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpor %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: incl %eax
; AVX512-NEXT: retq
bb:
%tmp = load <16 x i8>, ptr %arg, align 1
diff --git a/llvm/test/CodeGen/X86/vector-compress.ll b/llvm/test/CodeGen/X86/vector-compress.ll
index 22c8cd692400c..fe8613b4bc504 100644
--- a/llvm/test/CodeGen/X86/vector-compress.ll
+++ b/llvm/test/CodeGen/X86/vector-compress.ll
@@ -7,35 +7,37 @@ define <4 x i32> @test_compress_v4i32(<4 x i32> %vec, <4 x i1> %mask, <4 x i32>
; AVX2-LABEL: test_compress_v4i32:
; AVX2: # %bb.0:
; AVX2-NEXT: vpslld $31, %xmm1, %xmm1
-; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1
-; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrd $1, %xmm1, %eax
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: movl %esi, %edi
-; AVX2-NEXT: subl %eax, %edi
-; AVX2-NEXT: vpextrd $2, %xmm1, %edx
-; AVX2-NEXT: subl %edx, %edi
-; AVX2-NEXT: vpextrd $3, %xmm1, %ecx
-; AVX2-NEXT: subl %ecx, %edi
-; AVX2-NEXT: andl $3, %edi
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rsi, %rax
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rax, %rdx
+; AVX2-NEXT: vpsrad $31, %xmm1, %xmm3
+; AVX2-NEXT: vmovdqa %xmm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpsrld $31, %xmm1, %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: andl $3, %eax
+; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vmovd %xmm3, %edx
+; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: vpextrd $2, %xmm3, %esi
+; AVX2-NEXT: andl $1, %esi
+; AVX2-NEXT: addq %rcx, %rsi
+; AVX2-NEXT: vpextrd $3, %xmm3, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: addq %rsi, %rdi
; AVX2-NEXT: vextractps $3, %xmm0, %r8d
-; AVX2-NEXT: cmpq $4, %rcx
-; AVX2-NEXT: cmovbl -24(%rsp,%rdi,4), %r8d
+; AVX2-NEXT: cmpq $4, %rdi
+; AVX2-NEXT: cmovbl -24(%rsp,%rax,4), %r8d
; AVX2-NEXT: vmovss %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rsi,4)
-; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rax,4)
-; AVX2-NEXT: andl $3, %edx
-; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rdx,4)
-; AVX2-NEXT: cmpq $3, %rcx
+; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rdx,4)
+; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx,4)
+; AVX2-NEXT: andl $3, %esi
+; AVX2-NEXT: vextractps $3, %xmm0, -24(%rsp,%rsi,4)
+; AVX2-NEXT: cmpq $3, %rdi
; AVX2-NEXT: movl $3, %eax
-; AVX2-NEXT: cmovbq %rcx, %rax
+; AVX2-NEXT: cmovbq %rdi, %rax
; AVX2-NEXT: movl %r8d, -24(%rsp,%rax,4)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
@@ -68,40 +70,42 @@ define <4 x float> @test_compress_v4f32(<4 x float> %vec, <4 x i1> %mask, <4 x f
; AVX2-LABEL: test_compress_v4f32:
; AVX2: # %bb.0:
; AVX2-NEXT: vpslld $31, %xmm1, %xmm1
-; AVX2-NEXT: vpsrad $31, %xmm1, %xmm1
-; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrd $1, %xmm1, %edx
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: movl %esi, %edi
-; AVX2-NEXT: subl %edx, %edi
-; AVX2-NEXT: vpextrd $2, %xmm1, %ecx
-; AVX2-NEXT: subl %ecx, %edi
-; AVX2-NEXT: vpextrd $3, %xmm1, %eax
-; AVX2-NEXT: subl %eax, %edi
-; AVX2-NEXT: andl $3, %edi
+; AVX2-NEXT: vpsrad $31, %xmm1, %xmm3
+; AVX2-NEXT: vmovdqa %xmm2, -{{[0-9]+}}(%rsp)
+; AVX2-NEXT: vpsrld $31, %xmm1, %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: andl $3, %eax
; AVX2-NEXT: vmovss {{.*#+}} xmm1 = mem[0],zero,zero,zero
; AVX2-NEXT: vmovss %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rsi,4)
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %rsi, %rdx
-; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rdx,4)
+; AVX2-NEXT: vmovd %xmm3, %eax
+; AVX2-NEXT: andl $1, %eax
+; AVX2-NEXT: vextractps $1, %xmm0, -24(%rsp,%rax,4)
+; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: vextractps $2, %xmm0, -24(%rsp,%rcx,4)
+; AVX2-NEXT: vpextrd $2, %xmm3, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
-; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vpextrd $3, %xmm3, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rax, %rcx
+; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
+; AVX2-NEXT: andl $3, %eax
; AVX2-NEXT: vshufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rcx,4)
-; AVX2-NEXT: cmpq $3, %rax
-; AVX2-NEXT: movl $3, %ecx
-; AVX2-NEXT: cmovbq %rax, %rcx
+; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rax,4)
+; AVX2-NEXT: cmpq $3, %rcx
+; AVX2-NEXT: movl $3, %eax
+; AVX2-NEXT: cmovbq %rcx, %rax
; AVX2-NEXT: ja .LBB1_2
; AVX2-NEXT: # %bb.1:
; AVX2-NEXT: vmovaps %xmm1, %xmm0
; AVX2-NEXT: .LBB1_2:
-; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rcx,4)
+; AVX2-NEXT: vmovss %xmm0, -24(%rsp,%rax,4)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: retq
;
@@ -251,13 +255,13 @@ define <8 x i32> @test_compress_v8i32(<8 x i32> %vec, <8 x i1> %mask, <8 x i32>
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
; AVX2-NEXT: vpslld $31, %ymm2, %ymm2
; AVX2-NEXT: vpsrld $31, %ymm2, %ymm2
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpsadbw %ymm4, %ymm2, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4
-; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vpaddq %xmm4, %xmm2, %xmm2
; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %ecx
-; AVX2-NEXT: vmovd %xmm2, %eax
-; AVX2-NEXT: addl %ecx, %eax
+; AVX2-NEXT: vpaddq %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vmovq %xmm2, %rax
; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vpextrd $1, %xmm3, %ecx
; AVX2-NEXT: andl $1, %ecx
@@ -347,14 +351,14 @@ define <8 x float> @test_compress_v8f32(<8 x float> %vec, <8 x i1> %mask, <8 x f
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
; AVX2-NEXT: vpslld $31, %ymm2, %ymm2
; AVX2-NEXT: vpsrld $31, %ymm2, %ymm2
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpsadbw %ymm4, %ymm2, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4
-; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vpaddq %xmm4, %xmm2, %xmm2
; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpextrd $1, %xmm2, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vpaddq %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vmovq %xmm2, %rax
+; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vmovss {{.*#+}} xmm2 = mem[0],zero,zero,zero
; AVX2-NEXT: vmovss %xmm0, (%rsp)
; AVX2-NEXT: vmovd %xmm3, %eax
@@ -448,9 +452,9 @@ define <4 x i64> @test_compress_v4i64(<4 x i64> %vec, <4 x i1> %mask, <4 x i64>
; AVX2-NEXT: vpsrlq $63, %ymm1, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: andl $3, %eax
; AVX2-NEXT: vpextrq $1, %xmm1, %rcx
; AVX2-NEXT: vmovq %xmm1, %rdx
@@ -520,10 +524,10 @@ define <4 x double> @test_compress_v4f64(<4 x double> %vec, <4 x i1> %mask, <4 x
; AVX2-NEXT: vpsrlq $63, %ymm3, %ymm1
; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
; AVX2-NEXT: vpaddq %xmm2, %xmm1, %xmm1
-; AVX2-NEXT: vpextrq $1, %xmm1, %rax
-; AVX2-NEXT: vmovq %xmm1, %rcx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $3, %ecx
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vmovq %xmm1, %rax
+; AVX2-NEXT: andl $3, %eax
; AVX2-NEXT: vmovsd {{.*#+}} xmm1 = mem[0],zero
; AVX2-NEXT: vmovlpd %xmm0, (%rsp)
; AVX2-NEXT: vmovq %xmm3, %rax
@@ -600,15 +604,15 @@ define <16 x i32> @test_compress_v16i32(<16 x i32> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpsadbw %ymm4, %ymm3, %ymm3
; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $1, %xmm3, %eax
-; AVX2-NEXT: vmovd %xmm3, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $15, %ecx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vmovq %xmm3, %rax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $1, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
@@ -749,14 +753,14 @@ define <16 x float> @test_compress_v16f32(<16 x float> %vec, <16 x i1> %mask, <1
; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpsadbw %ymm4, %ymm3, %ymm3
; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpextrd $1, %xmm3, %eax
-; AVX2-NEXT: vmovd %xmm3, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vmovq %xmm3, %rax
+; AVX2-NEXT: andl $15, %eax
; AVX2-NEXT: vmovss {{.*#+}} xmm3 = mem[0],zero,zero,zero
; AVX2-NEXT: vmovss %xmm0, (%rsp)
; AVX2-NEXT: vmovd %xmm2, %eax
@@ -899,9 +903,9 @@ define <8 x i64> @test_compress_v8i64(<8 x i64> %vec, <8 x i1> %mask, <8 x i64>
; AVX2-NEXT: vpaddq %ymm3, %ymm4, %ymm3
; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm3, %rcx
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
; AVX2-NEXT: vmovq %xmm3, %rax
-; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vpextrw $1, %xmm2, %ecx
; AVX2-NEXT: andl $1, %ecx
@@ -991,10 +995,10 @@ define <8 x double> @test_compress_v8f64(<8 x double> %vec, <8 x i1> %mask, <8 x
; AVX2-NEXT: vpaddq %ymm3, %ymm4, %ymm3
; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vpextrq $1, %xmm3, %rax
-; AVX2-NEXT: vmovq %xmm3, %rcx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $7, %ecx
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
+; AVX2-NEXT: vmovq %xmm3, %rax
+; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vmovsd {{.*#+}} xmm3 = mem[0],zero
; AVX2-NEXT: vmovlps %xmm0, (%rsp)
; AVX2-NEXT: vmovd %xmm2, %eax
@@ -1076,139 +1080,109 @@ define <8 x double> @test_compress_v8f64(<8 x double> %vec, <8 x i1> %mask, <8 x
define <16 x i8> @test_compress_v16i8(<16 x i8> %vec, <16 x i1> %mask, <16 x i8> %passthru) nounwind {
; AVX2-LABEL: test_compress_v16i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: pushq %rbp
-; AVX2-NEXT: pushq %r15
-; AVX2-NEXT: pushq %r14
-; AVX2-NEXT: pushq %r13
-; AVX2-NEXT: pushq %r12
-; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: vpsllw $7, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: vpcmpgtb %xmm1, %xmm3, %xmm1
; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrb $1, %xmm1, %r13d
-; AVX2-NEXT: vmovd %xmm1, %esi
-; AVX2-NEXT: movl %esi, %eax
-; AVX2-NEXT: andb $1, %al
-; AVX2-NEXT: subb %r13b, %al
-; AVX2-NEXT: vpextrb $2, %xmm1, %edx
-; AVX2-NEXT: subb %dl, %al
-; AVX2-NEXT: vpextrb $3, %xmm1, %ebp
-; AVX2-NEXT: subb %bpl, %al
-; AVX2-NEXT: vpextrb $4, %xmm1, %r12d
-; AVX2-NEXT: subb %r12b, %al
-; AVX2-NEXT: vpextrb $5, %xmm1, %r15d
-; AVX2-NEXT: subb %r15b, %al
-; AVX2-NEXT: vpextrb $6, %xmm1, %r14d
-; AVX2-NEXT: subb %r14b, %al
-; AVX2-NEXT: vpextrb $7, %xmm1, %ebx
-; AVX2-NEXT: subb %bl, %al
-; AVX2-NEXT: vpextrb $8, %xmm1, %r11d
-; AVX2-NEXT: subb %r11b, %al
-; AVX2-NEXT: vpextrb $9, %xmm1, %r10d
-; AVX2-NEXT: subb %r10b, %al
-; AVX2-NEXT: vpextrb $10, %xmm1, %r9d
-; AVX2-NEXT: subb %r9b, %al
-; AVX2-NEXT: vpextrb $11, %xmm1, %r8d
-; AVX2-NEXT: subb %r8b, %al
-; AVX2-NEXT: vpextrb $12, %xmm1, %edi
-; AVX2-NEXT: subb %dil, %al
-; AVX2-NEXT: vpextrb $13, %xmm1, %ecx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: subb %cl, %al
-; AVX2-NEXT: vpextrb $14, %xmm1, %ecx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
-; AVX2-NEXT: subb %cl, %al
-; AVX2-NEXT: vpextrb $15, %xmm1, %ecx
-; AVX2-NEXT: subb %cl, %al
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vmovd %xmm2, %eax
; AVX2-NEXT: andl $15, %eax
-; AVX2-NEXT: movzbl -40(%rsp,%rax), %eax
-; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX2-NEXT: movzbl -24(%rsp,%rax), %eax
; AVX2-NEXT: vpextrb $0, %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: vpextrb $1, %xmm0, -40(%rsp,%rsi)
-; AVX2-NEXT: andl $1, %r13d
-; AVX2-NEXT: addq %rsi, %r13
-; AVX2-NEXT: vpextrb $2, %xmm0, -40(%rsp,%r13)
+; AVX2-NEXT: vmovd %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vpextrb $1, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $1, %xmm1, %edx
; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: addq %r13, %rdx
-; AVX2-NEXT: vpextrb $3, %xmm0, -40(%rsp,%rdx)
-; AVX2-NEXT: andl $1, %ebp
-; AVX2-NEXT: addq %rdx, %rbp
-; AVX2-NEXT: vpextrb $4, %xmm0, -40(%rsp,%rbp)
-; AVX2-NEXT: andl $1, %r12d
-; AVX2-NEXT: addq %rbp, %r12
-; AVX2-NEXT: andl $1, %r15d
-; AVX2-NEXT: addq %r12, %r15
-; AVX2-NEXT: # kill: def $r12d killed $r12d killed $r12 def $r12
-; AVX2-NEXT: andl $15, %r12d
-; AVX2-NEXT: vpextrb $5, %xmm0, -40(%rsp,%r12)
-; AVX2-NEXT: andl $1, %r14d
-; AVX2-NEXT: addq %r15, %r14
-; AVX2-NEXT: # kill: def $r15d killed $r15d killed $r15 def $r15
-; AVX2-NEXT: andl $15, %r15d
-; AVX2-NEXT: vpextrb $6, %xmm0, -40(%rsp,%r15)
-; AVX2-NEXT: andl $1, %ebx
-; AVX2-NEXT: addq %r14, %rbx
-; AVX2-NEXT: # kill: def $r14d killed $r14d killed $r14 def $r14
-; AVX2-NEXT: andl $15, %r14d
-; AVX2-NEXT: vpextrb $7, %xmm0, -40(%rsp,%r14)
-; AVX2-NEXT: andl $1, %r11d
-; AVX2-NEXT: addq %rbx, %r11
-; AVX2-NEXT: # kill: def $ebx killed $ebx killed $rbx def $rbx
-; AVX2-NEXT: andl $15, %ebx
-; AVX2-NEXT: vpextrb $8, %xmm0, -40(%rsp,%rbx)
-; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r11, %r10
-; AVX2-NEXT: # kill: def $r11d killed $r11d killed $r11 def $r11
-; AVX2-NEXT: andl $15, %r11d
-; AVX2-NEXT: vpextrb $9, %xmm0, -40(%rsp,%r11)
-; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: addq %r10, %r9
-; AVX2-NEXT: # kill: def $r10d killed $r10d killed $r10 def $r10
-; AVX2-NEXT: andl $15, %r10d
-; AVX2-NEXT: vpextrb $10, %xmm0, -40(%rsp,%r10)
-; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %r9, %r8
-; AVX2-NEXT: # kill: def $r9d killed $r9d killed $r9 def $r9
-; AVX2-NEXT: andl $15, %r9d
-; AVX2-NEXT: vpextrb $11, %xmm0, -40(%rsp,%r9)
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: addq %r8, %rdi
-; AVX2-NEXT: # kill: def $r8d killed $r8d killed $r8 def $r8
-; AVX2-NEXT: andl $15, %r8d
-; AVX2-NEXT: vpextrb $12, %xmm0, -40(%rsp,%r8)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rsi # 8-byte Reload
-; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: addq %rdi, %rsi
-; AVX2-NEXT: # kill: def $edi killed $edi killed $rdi def $rdi
-; AVX2-NEXT: andl $15, %edi
-; AVX2-NEXT: vpextrb $13, %xmm0, -40(%rsp,%rdi)
-; AVX2-NEXT: movq {{[-0-9]+}}(%r{{[sb]}}p), %rax # 8-byte Reload
-; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %rsi, %rax
-; AVX2-NEXT: # kill: def $esi killed $esi killed $rsi def $rsi
-; AVX2-NEXT: andl $15, %esi
-; AVX2-NEXT: vpextrb $14, %xmm0, -40(%rsp,%rsi)
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: vpextrb $2, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $2, %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: addq %rax, %rcx
-; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
-; AVX2-NEXT: andl $15, %eax
-; AVX2-NEXT: vpextrb $15, %xmm0, -40(%rsp,%rax)
-; AVX2-NEXT: cmpq $15, %rcx
-; AVX2-NEXT: movl $15, %eax
-; AVX2-NEXT: cmovbq %rcx, %rax
-; AVX2-NEXT: vpextrb $15, %xmm0, %ecx
-; AVX2-NEXT: cmovbel {{[-0-9]+}}(%r{{[sb]}}p), %ecx # 4-byte Folded Reload
-; AVX2-NEXT: movb %cl, -40(%rsp,%rax)
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: vpextrb $3, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $3, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: vpextrb $4, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $4, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: vpextrb $5, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $5, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $6, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
+; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vpextrb $6, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $7, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $7, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $8, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
+; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vpextrb $8, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $9, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $9, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $10, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
+; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vpextrb $10, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $11, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $11, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $12, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
+; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vpextrb $12, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $13, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $13, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: vpextrb $14, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: # kill: def $edx killed $edx killed $rdx def $rdx
+; AVX2-NEXT: andl $15, %edx
+; AVX2-NEXT: vpextrb $14, %xmm0, -24(%rsp,%rdx)
+; AVX2-NEXT: vpextrb $15, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: addq %rcx, %rdx
+; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
+; AVX2-NEXT: andl $15, %ecx
+; AVX2-NEXT: vpextrb $15, %xmm0, -24(%rsp,%rcx)
+; AVX2-NEXT: cmpq $15, %rdx
+; AVX2-NEXT: movl $15, %ecx
+; AVX2-NEXT: cmovbq %rdx, %rcx
+; AVX2-NEXT: vpextrb $15, %xmm0, %edx
+; AVX2-NEXT: cmovbel %eax, %edx
+; AVX2-NEXT: movb %dl, -24(%rsp,%rcx)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
-; AVX2-NEXT: popq %rbx
-; AVX2-NEXT: popq %r12
-; AVX2-NEXT: popq %r13
-; AVX2-NEXT: popq %r14
-; AVX2-NEXT: popq %r15
-; AVX2-NEXT: popq %rbp
; AVX2-NEXT: retq
;
; AVX512F-LABEL: test_compress_v16i8:
@@ -1243,55 +1217,61 @@ define <8 x i16> @test_compress_v8i16(<8 x i16> %vec, <8 x i1> %mask, <8 x i16>
; AVX2-NEXT: vmovaps %xmm2, -{{[0-9]+}}(%rsp)
; AVX2-NEXT: vpextrw $1, %xmm1, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: vmovd %xmm1, %ecx
-; AVX2-NEXT: andl $1, %ecx
-; AVX2-NEXT: leal (%rcx,%rax), %esi
-; AVX2-NEXT: vpextrw $2, %xmm1, %edi
-; AVX2-NEXT: andl $1, %edi
-; AVX2-NEXT: vpextrw $3, %xmm1, %edx
-; AVX2-NEXT: andl $1, %edx
-; AVX2-NEXT: leal (%rdi,%rdx), %r10d
-; AVX2-NEXT: addl %esi, %r10d
-; AVX2-NEXT: vpextrw $4, %xmm1, %r9d
-; AVX2-NEXT: andl $1, %r9d
-; AVX2-NEXT: vpextrw $5, %xmm1, %esi
+; AVX2-NEXT: vmovd %xmm1, %esi
; AVX2-NEXT: andl $1, %esi
-; AVX2-NEXT: leal (%r9,%rsi), %r11d
-; AVX2-NEXT: vpextrw $6, %xmm1, %r8d
-; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addl %r8d, %r11d
-; AVX2-NEXT: addl %r10d, %r11d
-; AVX2-NEXT: vpextrw $7, %xmm1, %r10d
+; AVX2-NEXT: vmovd %esi, %xmm2
+; AVX2-NEXT: vpinsrw $1, %eax, %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $2, %xmm1, %r10d
; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addl %r10d, %r11d
-; AVX2-NEXT: andl $7, %r11d
-; AVX2-NEXT: addq %rcx, %rax
-; AVX2-NEXT: addq %rax, %rdi
+; AVX2-NEXT: vpinsrw $2, %r10d, %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $3, %xmm1, %r9d
+; AVX2-NEXT: andl $1, %r9d
+; AVX2-NEXT: vpinsrw $3, %r9d, %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $4, %xmm1, %r8d
+; AVX2-NEXT: andl $1, %r8d
+; AVX2-NEXT: vpinsrw $4, %r8d, %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $5, %xmm1, %edi
+; AVX2-NEXT: andl $1, %edi
+; AVX2-NEXT: vpinsrw $5, %edi, %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $6, %xmm1, %edx
+; AVX2-NEXT: andl $1, %edx
+; AVX2-NEXT: vpinsrw $6, %edx, %xmm2, %xmm2
+; AVX2-NEXT: vpextrw $7, %xmm1, %ecx
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: vpinsrw $7, %ecx, %xmm2, %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vmovq %xmm1, %rbx
+; AVX2-NEXT: andl $7, %ebx
+; AVX2-NEXT: addq %rsi, %rax
+; AVX2-NEXT: addq %rax, %r10
+; AVX2-NEXT: addq %r10, %r9
+; AVX2-NEXT: addq %r9, %r8
+; AVX2-NEXT: addq %r8, %rdi
; AVX2-NEXT: addq %rdi, %rdx
-; AVX2-NEXT: addq %rdx, %r9
-; AVX2-NEXT: addq %r9, %rsi
-; AVX2-NEXT: addq %rsi, %r8
-; AVX2-NEXT: addq %r8, %r10
-; AVX2-NEXT: vpextrw $7, %xmm0, %ebx
-; AVX2-NEXT: cmpq $8, %r10
-; AVX2-NEXT: cmovbw -16(%rsp,%r11,2), %bx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: vpextrw $7, %xmm0, %r11d
+; AVX2-NEXT: cmpq $8, %rcx
+; AVX2-NEXT: cmovbw -16(%rsp,%rbx,2), %r11w
; AVX2-NEXT: vpextrw $0, %xmm0, -{{[0-9]+}}(%rsp)
-; AVX2-NEXT: vpextrw $1, %xmm0, -16(%rsp,%rcx,2)
+; AVX2-NEXT: vpextrw $1, %xmm0, -16(%rsp,%rsi,2)
; AVX2-NEXT: vpextrw $2, %xmm0, -16(%rsp,%rax,2)
-; AVX2-NEXT: vpextrw $3, %xmm0, -16(%rsp,%rdi,2)
-; AVX2-NEXT: andl $7, %edx
-; AVX2-NEXT: vpextrw $4, %xmm0, -16(%rsp,%rdx,2)
+; AVX2-NEXT: vpextrw $3, %xmm0, -16(%rsp,%r10,2)
; AVX2-NEXT: andl $7, %r9d
-; AVX2-NEXT: vpextrw $5, %xmm0, -16(%rsp,%r9,2)
-; AVX2-NEXT: andl $7, %esi
-; AVX2-NEXT: vpextrw $6, %xmm0, -16(%rsp,%rsi,2)
+; AVX2-NEXT: vpextrw $4, %xmm0, -16(%rsp,%r9,2)
; AVX2-NEXT: andl $7, %r8d
-; AVX2-NEXT: vpextrw $7, %xmm0, -16(%rsp,%r8,2)
-; AVX2-NEXT: cmpq $7, %r10
+; AVX2-NEXT: vpextrw $5, %xmm0, -16(%rsp,%r8,2)
+; AVX2-NEXT: andl $7, %edi
+; AVX2-NEXT: vpextrw $6, %xmm0, -16(%rsp,%rdi,2)
+; AVX2-NEXT: andl $7, %edx
+; AVX2-NEXT: vpextrw $7, %xmm0, -16(%rsp,%rdx,2)
+; AVX2-NEXT: cmpq $7, %rcx
; AVX2-NEXT: movl $7, %eax
-; AVX2-NEXT: cmovbq %r10, %rax
+; AVX2-NEXT: cmovbq %rcx, %rax
; AVX2-NEXT: movl %eax, %eax
-; AVX2-NEXT: movw %bx, -16(%rsp,%rax,2)
+; AVX2-NEXT: movw %r11w, -16(%rsp,%rax,2)
; AVX2-NEXT: vmovaps -{{[0-9]+}}(%rsp), %xmm0
; AVX2-NEXT: popq %rbx
; AVX2-NEXT: retq
@@ -1335,16 +1315,11 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm4
; AVX2-NEXT: vpand %xmm2, %xmm3, %xmm2
; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpshufb {{.*#+}} xmm4 = xmm2[8,9,10,11,12,13,14,15,12,13],zero,zero,xmm2[14,15],zero,zero
-; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[1,1,1,1]
-; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpsrld $16, %xmm2, %xmm4
+; AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
; AVX2-NEXT: vpaddb %xmm4, %xmm2, %xmm2
-; AVX2-NEXT: vpextrb $1, %xmm2, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: addb %al, %cl
-; AVX2-NEXT: movzbl %cl, %eax
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpsadbw %xmm4, %xmm2, %xmm2
+; AVX2-NEXT: vmovd %xmm2, %eax
; AVX2-NEXT: andl $31, %eax
; AVX2-NEXT: movzbl (%rsp,%rax), %eax
; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp)
@@ -1527,13 +1502,12 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $31, %ecx
; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: vpextrb $15, %xmm0, %ecx
-; AVX2-NEXT: cmpq $32, %rdx
-; AVX2-NEXT: cmovbl %eax, %ecx
; AVX2-NEXT: cmpq $31, %rdx
-; AVX2-NEXT: movl $31, %eax
-; AVX2-NEXT: cmovbq %rdx, %rax
-; AVX2-NEXT: movb %cl, (%rsp,%rax)
+; AVX2-NEXT: movl $31, %ecx
+; AVX2-NEXT: cmovbq %rdx, %rcx
+; AVX2-NEXT: vpextrb $15, %xmm0, %edx
+; AVX2-NEXT: cmovbel %eax, %edx
+; AVX2-NEXT: movb %dl, (%rsp,%rcx)
; AVX2-NEXT: vmovaps (%rsp), %ymm0
; AVX2-NEXT: movq %rbp, %rsp
; AVX2-NEXT: popq %rbp
@@ -1556,21 +1530,16 @@ define <32 x i8> @test_compress_v32i8(<32 x i8> %vec, <32 x i1> %mask, <32 x i8>
; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k2} {z}
; AVX512F-NEXT: vpmovdb %zmm3, (%rsp)
; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm3
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
-; AVX512F-NEXT: vextracti64x4 $1, %zmm3, %ymm4
-; AVX512F-NEXT: vpaddd %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX512F-NEXT: vpaddd %xmm4, %xmm3, %xmm3
+; AVX512F-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512F-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
; AVX512F-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT: vpextrd $1, %xmm3, %eax
-; AVX512F-NEXT: vmovd %xmm3, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vpaddq %xmm4, %xmm3, %xmm3
+; AVX512F-NEXT: vmovq %xmm3, %rax
+; AVX512F-NEXT: andl $31, %eax
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vpmovdb %zmm0, (%rsp,%rcx)
+; AVX512F-NEXT: vpmovdb %zmm0, (%rsp,%rax)
; AVX512F-NEXT: vpsllw $7, %ymm1, %ymm0
; AVX512F-NEXT: vpblendvb %ymm0, (%rsp), %ymm2, %ymm0
; AVX512F-NEXT: movq %rbp, %rsp
@@ -1605,17 +1574,15 @@ define <16 x i16> @test_compress_v16i16(<16 x i16> %vec, <16 x i1> %mask, <16 x
; AVX2-NEXT: vpsraw $15, %ymm3, %ymm1
; AVX2-NEXT: vmovaps %ymm2, (%rsp)
; AVX2-NEXT: vpsrlw $15, %ymm3, %ymm2
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpsadbw %ymm3, %ymm2, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
-; AVX2-NEXT: vpaddw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
-; AVX2-NEXT: vpaddw %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[1,1,1,1]
-; AVX2-NEXT: vpaddw %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpextrw $1, %xmm2, %eax
-; AVX2-NEXT: vmovd %xmm2, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $15, %ecx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vmovq %xmm2, %rax
+; AVX2-NEXT: andl $15, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrw $1, %xmm1, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm1, %ecx
@@ -1755,15 +1722,15 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: pushq %rbx
; AVX2-NEXT: andq $-32, %rsp
; AVX2-NEXT: subq $96, %rsp
-; AVX2-NEXT: movl %r9d, %r10d
-; AVX2-NEXT: movl %r8d, %r9d
-; AVX2-NEXT: movl %ecx, %r8d
+; AVX2-NEXT: # kill: def $r9d killed $r9d def $r9
+; AVX2-NEXT: # kill: def $r8d killed $r8d def $r8
+; AVX2-NEXT: # kill: def $ecx killed $ecx def $rcx
; AVX2-NEXT: # kill: def $edx killed $edx def $rdx
; AVX2-NEXT: # kill: def $esi killed $esi def $rsi
; AVX2-NEXT: # kill: def $edi killed $edi def $rdi
; AVX2-NEXT: movzbl 360(%rbp), %eax
-; AVX2-NEXT: movzbl 352(%rbp), %ecx
-; AVX2-NEXT: vmovd %ecx, %xmm4
+; AVX2-NEXT: movzbl 352(%rbp), %r10d
+; AVX2-NEXT: vmovd %r10d, %xmm4
; AVX2-NEXT: vpinsrb $1, %eax, %xmm4, %xmm4
; AVX2-NEXT: movzbl 368(%rbp), %eax
; AVX2-NEXT: vpinsrb $2, %eax, %xmm4, %xmm4
@@ -1829,83 +1796,77 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: vmovd %edi, %xmm5
; AVX2-NEXT: vpinsrb $1, %esi, %xmm5, %xmm5
; AVX2-NEXT: vpinsrb $2, %edx, %xmm5, %xmm5
-; AVX2-NEXT: vpinsrb $3, %r8d, %xmm5, %xmm5
-; AVX2-NEXT: vpinsrb $4, %r9d, %xmm5, %xmm5
-; AVX2-NEXT: vpinsrb $5, %r10d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 16(%rbp), %r11d
-; AVX2-NEXT: vpinsrb $6, %r11d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 24(%rbp), %ebx
-; AVX2-NEXT: vpinsrb $7, %ebx, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 32(%rbp), %r14d
-; AVX2-NEXT: vpinsrb $8, %r14d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 40(%rbp), %r15d
-; AVX2-NEXT: vpinsrb $9, %r15d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 48(%rbp), %r12d
-; AVX2-NEXT: vpinsrb $10, %r12d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 56(%rbp), %r13d
-; AVX2-NEXT: vpinsrb $11, %r13d, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 64(%rbp), %eax
-; AVX2-NEXT: vpinsrb $12, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 72(%rbp), %eax
-; AVX2-NEXT: vpinsrb $13, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 80(%rbp), %eax
-; AVX2-NEXT: vpinsrb $14, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 88(%rbp), %eax
-; AVX2-NEXT: vpinsrb $15, %eax, %xmm5, %xmm5
-; AVX2-NEXT: movzbl 96(%rbp), %eax
-; AVX2-NEXT: vmovd %eax, %xmm6
-; AVX2-NEXT: movzbl 104(%rbp), %eax
-; AVX2-NEXT: vpinsrb $1, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 112(%rbp), %eax
-; AVX2-NEXT: vpinsrb $2, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 120(%rbp), %eax
-; AVX2-NEXT: vpinsrb $3, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 128(%rbp), %eax
-; AVX2-NEXT: vpinsrb $4, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 136(%rbp), %eax
-; AVX2-NEXT: vpinsrb $5, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 144(%rbp), %eax
-; AVX2-NEXT: vpinsrb $6, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 152(%rbp), %eax
-; AVX2-NEXT: vpinsrb $7, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 160(%rbp), %eax
-; AVX2-NEXT: vpinsrb $8, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 168(%rbp), %eax
-; AVX2-NEXT: vpinsrb $9, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 176(%rbp), %eax
-; AVX2-NEXT: vpinsrb $10, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 184(%rbp), %eax
-; AVX2-NEXT: vpinsrb $11, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 192(%rbp), %eax
-; AVX2-NEXT: vpinsrb $12, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 200(%rbp), %eax
-; AVX2-NEXT: vpinsrb $13, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 208(%rbp), %eax
-; AVX2-NEXT: vpinsrb $14, %eax, %xmm6, %xmm6
-; AVX2-NEXT: movzbl 216(%rbp), %eax
-; AVX2-NEXT: vpinsrb $15, %eax, %xmm6, %xmm6
+; AVX2-NEXT: vpinsrb $3, %ecx, %xmm5, %xmm5
+; AVX2-NEXT: vpinsrb $4, %r8d, %xmm5, %xmm5
+; AVX2-NEXT: vpinsrb $5, %r9d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 16(%rbp), %eax
+; AVX2-NEXT: vpinsrb $6, %eax, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 24(%rbp), %r10d
+; AVX2-NEXT: vpinsrb $7, %r10d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 32(%rbp), %r11d
+; AVX2-NEXT: vpinsrb $8, %r11d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 40(%rbp), %ebx
+; AVX2-NEXT: vpinsrb $9, %ebx, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 48(%rbp), %r14d
+; AVX2-NEXT: vpinsrb $10, %r14d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 56(%rbp), %r15d
+; AVX2-NEXT: vpinsrb $11, %r15d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 64(%rbp), %r12d
+; AVX2-NEXT: vpinsrb $12, %r12d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 72(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $13, %r13d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 80(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $14, %r13d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 88(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $15, %r13d, %xmm5, %xmm5
+; AVX2-NEXT: movzbl 96(%rbp), %r13d
+; AVX2-NEXT: vmovd %r13d, %xmm6
+; AVX2-NEXT: movzbl 104(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $1, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 112(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $2, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 120(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $3, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 128(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $4, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 136(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $5, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 144(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $6, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 152(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $7, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 160(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $8, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 168(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $9, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 176(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $10, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 184(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $11, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 192(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $12, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 200(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $13, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 208(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $14, %r13d, %xmm6, %xmm6
+; AVX2-NEXT: movzbl 216(%rbp), %r13d
+; AVX2-NEXT: vpinsrb $15, %r13d, %xmm6, %xmm6
; AVX2-NEXT: vinserti128 $1, %xmm6, %ymm5, %ymm5
-; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm6 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
-; AVX2-NEXT: vpand %ymm6, %ymm5, %ymm5
-; AVX2-NEXT: vpand %ymm6, %ymm4, %ymm4
-; AVX2-NEXT: vpaddb %ymm4, %ymm5, %ymm4
-; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpshufb {{.*#+}} xmm5 = xmm4[8,9,10,11,12,13,14,15,12,13],zero,zero,xmm4[14,15],zero,zero
-; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
-; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpsrld $16, %xmm4, %xmm5
-; AVX2-NEXT: vpaddb %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpextrb $1, %xmm4, %ecx
-; AVX2-NEXT: vmovd %xmm4, %eax
-; AVX2-NEXT: addb %cl, %al
; AVX2-NEXT: vmovaps %ymm3, {{[0-9]+}}(%rsp)
; AVX2-NEXT: vmovaps %ymm2, (%rsp)
-; AVX2-NEXT: movzbl %al, %eax
-; AVX2-NEXT: andl $63, %eax
-; AVX2-NEXT: movzbl (%rsp,%rax), %eax
-; AVX2-NEXT: movl %eax, {{[-0-9]+}}(%r{{[sb]}}p) # 4-byte Spill
+; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm2 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
+; AVX2-NEXT: vpand %ymm2, %ymm5, %ymm3
+; AVX2-NEXT: vpand %ymm2, %ymm4, %ymm2
+; AVX2-NEXT: vpaddb %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
+; AVX2-NEXT: vpaddb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpaddb %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; AVX2-NEXT: vmovd %xmm2, %r13d
+; AVX2-NEXT: andl $63, %r13d
+; AVX2-NEXT: movzbl (%rsp,%r13), %r13d
; AVX2-NEXT: vpextrb $0, %xmm0, (%rsp)
; AVX2-NEXT: andl $1, %edi
; AVX2-NEXT: vpextrb $1, %xmm0, (%rsp,%rdi)
@@ -1915,53 +1876,52 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: andl $1, %edx
; AVX2-NEXT: addq %rsi, %rdx
; AVX2-NEXT: vpextrb $3, %xmm0, (%rsp,%rdx)
+; AVX2-NEXT: andl $1, %ecx
+; AVX2-NEXT: addq %rdx, %rcx
+; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%rcx)
; AVX2-NEXT: andl $1, %r8d
-; AVX2-NEXT: addq %rdx, %r8
-; AVX2-NEXT: vpextrb $4, %xmm0, (%rsp,%r8)
+; AVX2-NEXT: addq %rcx, %r8
+; AVX2-NEXT: movl %r8d, %ecx
+; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rcx)
; AVX2-NEXT: andl $1, %r9d
; AVX2-NEXT: addq %r8, %r9
-; AVX2-NEXT: movl %r9d, %eax
-; AVX2-NEXT: vpextrb $5, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: andl $1, %r10d
-; AVX2-NEXT: addq %r9, %r10
-; AVX2-NEXT: movzbl %r11b, %eax
+; AVX2-NEXT: movzbl %al, %eax
; AVX2-NEXT: andl $1, %eax
-; AVX2-NEXT: addq %r10, %rax
-; AVX2-NEXT: # kill: def $r10d killed $r10d killed $r10 def $r10
-; AVX2-NEXT: andl $63, %r10d
-; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%r10)
-; AVX2-NEXT: movzbl %bl, %ecx
+; AVX2-NEXT: addq %r9, %rax
+; AVX2-NEXT: # kill: def $r9d killed $r9d killed $r9 def $r9
+; AVX2-NEXT: andl $63, %r9d
+; AVX2-NEXT: vpextrb $6, %xmm0, (%rsp,%r9)
+; AVX2-NEXT: movzbl %r10b, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vpextrb $7, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl %r14b, %eax
+; AVX2-NEXT: movzbl %r11b, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vpextrb $8, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl %r15b, %ecx
+; AVX2-NEXT: movzbl %bl, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vpextrb $9, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl %r12b, %eax
+; AVX2-NEXT: movzbl %r14b, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
; AVX2-NEXT: andl $63, %ecx
; AVX2-NEXT: vpextrb $10, %xmm0, (%rsp,%rcx)
-; AVX2-NEXT: movzbl %r13b, %ecx
+; AVX2-NEXT: movzbl %r15b, %ecx
; AVX2-NEXT: andl $1, %ecx
; AVX2-NEXT: addq %rax, %rcx
; AVX2-NEXT: # kill: def $eax killed $eax killed $rax def $rax
; AVX2-NEXT: andl $63, %eax
; AVX2-NEXT: vpextrb $11, %xmm0, (%rsp,%rax)
-; AVX2-NEXT: movzbl 64(%rbp), %eax
-; AVX2-NEXT: movzbl %al, %eax
+; AVX2-NEXT: movzbl %r12b, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: addq %rcx, %rax
; AVX2-NEXT: # kill: def $ecx killed $ecx killed $rcx def $rcx
@@ -2328,7 +2288,7 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX2-NEXT: vpextrb $15, %xmm0, (%rsp,%rax)
; AVX2-NEXT: vpextrb $15, %xmm0, %eax
; AVX2-NEXT: cmpq $64, %rcx
-; AVX2-NEXT: cmovbl {{[-0-9]+}}(%r{{[sb]}}p), %eax # 4-byte Folded Reload
+; AVX2-NEXT: cmovbl %r13d, %eax
; AVX2-NEXT: cmpq $63, %rcx
; AVX2-NEXT: movl $63, %edx
; AVX2-NEXT: cmovbq %rcx, %rdx
@@ -2429,39 +2389,38 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm2 {%k4} {z} = -1
; AVX512F-NEXT: vpsrld $31, %zmm2, %zmm3
; AVX512F-NEXT: vextracti64x4 $1, %zmm3, %ymm4
-; AVX512F-NEXT: vpaddd %ymm4, %ymm3, %ymm3
-; AVX512F-NEXT: vextracti128 $1, %ymm3, %xmm4
-; AVX512F-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm4, %xmm3, %xmm3
-; AVX512F-NEXT: vpextrd $1, %xmm3, %eax
-; AVX512F-NEXT: vmovd %xmm3, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vpaddb %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512F-NEXT: vpsadbw %ymm4, %ymm3, %ymm3
+; AVX512F-NEXT: vextracti128 $1, %ymm3, %xmm5
+; AVX512F-NEXT: vpaddq %xmm5, %xmm3, %xmm3
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
+; AVX512F-NEXT: vpaddq %xmm5, %xmm3, %xmm3
+; AVX512F-NEXT: vmovq %xmm3, %rax
+; AVX512F-NEXT: andl $31, %eax
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm3
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero,xmm3[8],zero,zero,zero,xmm3[9],zero,zero,zero,xmm3[10],zero,zero,zero,xmm3[11],zero,zero,zero,xmm3[12],zero,zero,zero,xmm3[13],zero,zero,zero,xmm3[14],zero,zero,zero,xmm3[15],zero,zero,zero
; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k2} {z}
-; AVX512F-NEXT: vpmovdb %zmm3, (%rsp,%rcx)
+; AVX512F-NEXT: vpmovdb %zmm3, (%rsp,%rax)
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm3 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; AVX512F-NEXT: vpcompressd %zmm3, %zmm3 {%k3} {z}
; AVX512F-NEXT: vpmovdb %zmm3, {{[0-9]+}}(%rsp)
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm3 {%k3} {z} = -1
-; AVX512F-NEXT: vpsrld $31, %zmm3, %zmm4
-; AVX512F-NEXT: vextracti64x4 $1, %zmm4, %ymm5
-; AVX512F-NEXT: vpaddd %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT: vpsrld $31, %zmm3, %zmm5
+; AVX512F-NEXT: vextracti64x4 $1, %zmm5, %ymm6
+; AVX512F-NEXT: vpaddb %ymm6, %ymm5, %ymm5
+; AVX512F-NEXT: vpsadbw %ymm4, %ymm5, %ymm4
; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX512F-NEXT: vpaddq %xmm5, %xmm4, %xmm4
; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
-; AVX512F-NEXT: vmovd %xmm4, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vpaddq %xmm5, %xmm4, %xmm4
+; AVX512F-NEXT: vmovq %xmm4, %rax
+; AVX512F-NEXT: andl $31, %eax
; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm0
; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vpmovdb %zmm0, 32(%rsp,%rcx)
+; AVX512F-NEXT: vpmovdb %zmm0, 32(%rsp,%rax)
; AVX512F-NEXT: vmovdqa (%rsp), %ymm0
; AVX512F-NEXT: vmovdqa %ymm0, {{[0-9]+}}(%rsp)
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1
@@ -2473,12 +2432,12 @@ define <64 x i8> @test_compress_v64i8(<64 x i8> %vec, <64 x i1> %mask, <64 x i8>
; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
-; AVX512F-NEXT: vmovd %xmm4, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $63, %ecx
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
+; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
+; AVX512F-NEXT: vmovd %xmm4, %eax
+; AVX512F-NEXT: andl $63, %eax
; AVX512F-NEXT: vmovdqa {{[0-9]+}}(%rsp), %ymm4
-; AVX512F-NEXT: vmovdqa %ymm4, 64(%rsp,%rcx)
+; AVX512F-NEXT: vmovdqa %ymm4, 64(%rsp,%rax)
; AVX512F-NEXT: vpmovdb %zmm3, %xmm3
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm4 {%k1} {z} = -1
; AVX512F-NEXT: vpmovdb %zmm4, %xmm4
@@ -2526,17 +2485,15 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm6 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero,xmm2[8],zero,xmm2[9],zero,xmm2[10],zero,xmm2[11],zero,xmm2[12],zero,xmm2[13],zero,xmm2[14],zero,xmm2[15],zero
; AVX2-NEXT: vpand %ymm5, %ymm6, %ymm5
; AVX2-NEXT: vpaddw %ymm4, %ymm5, %ymm4
+; AVX2-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX2-NEXT: vpsadbw %ymm5, %ymm4, %ymm4
; AVX2-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX2-NEXT: vpaddw %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vpaddq %xmm5, %xmm4, %xmm4
; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; AVX2-NEXT: vpaddw %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[1,1,1,1]
-; AVX2-NEXT: vpaddw %xmm5, %xmm4, %xmm4
-; AVX2-NEXT: vpextrw $1, %xmm4, %eax
-; AVX2-NEXT: vmovd %xmm4, %ecx
-; AVX2-NEXT: addl %eax, %ecx
-; AVX2-NEXT: andl $31, %ecx
-; AVX2-NEXT: movq %rcx, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
+; AVX2-NEXT: vpaddq %xmm5, %xmm4, %xmm4
+; AVX2-NEXT: vmovq %xmm4, %rax
+; AVX2-NEXT: andl $31, %eax
+; AVX2-NEXT: movq %rax, {{[-0-9]+}}(%r{{[sb]}}p) # 8-byte Spill
; AVX2-NEXT: vpextrb $1, %xmm2, %eax
; AVX2-NEXT: andl $1, %eax
; AVX2-NEXT: vmovd %xmm2, %ecx
@@ -2761,35 +2718,30 @@ define <32 x i16> @test_compress_v32i16(<32 x i16> %vec, <32 x i1> %mask, <32 x
; AVX512F-NEXT: andq $-64, %rsp
; AVX512F-NEXT: subq $128, %rsp
; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
-; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm4 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero,xmm5[8],zero,xmm5[9],zero,xmm5[10],zero,xmm5[11],zero,xmm5[12],zero,xmm5[13],zero,xmm5[14],zero,xmm5[15],zero
-; AVX512F-NEXT: vpmovsxbd %xmm5, %zmm5
-; AVX512F-NEXT: vpslld $31, %zmm5, %zmm5
-; AVX512F-NEXT: vptestmd %zmm5, %zmm5, %k1
-; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm5
-; AVX512F-NEXT: vpslld $31, %zmm5, %zmm5
-; AVX512F-NEXT: vptestmd %zmm5, %zmm5, %k2
-; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm5 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
-; AVX512F-NEXT: vpcompressd %zmm5, %zmm5 {%k2} {z}
-; AVX512F-NEXT: vpmovdw %zmm5, (%rsp)
+; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm4
+; AVX512F-NEXT: vpmovzxbw {{.*#+}} ymm5 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero,xmm4[8],zero,xmm4[9],zero,xmm4[10],zero,xmm4[11],zero,xmm4[12],zero,xmm4[13],zero,xmm4[14],zero,xmm4[15],zero
+; AVX512F-NEXT: vpmovsxbd %xmm4, %zmm4
+; AVX512F-NEXT: vpslld $31, %zmm4, %zmm4
+; AVX512F-NEXT: vptestmd %zmm4, %zmm4, %k1
+; AVX512F-NEXT: vpmovsxbd %xmm1, %zmm4
+; AVX512F-NEXT: vpslld $31, %zmm4, %zmm4
+; AVX512F-NEXT: vptestmd %zmm4, %zmm4, %k2
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm4 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
+; AVX512F-NEXT: vpcompressd %zmm4, %zmm4 {%k2} {z}
+; AVX512F-NEXT: vpmovdw %zmm4, (%rsp)
; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX512F-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
-; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm5
-; AVX512F-NEXT: vpaddd %ymm5, %ymm1, %ymm1
-; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX512F-NEXT: vpaddd %xmm5, %xmm1, %xmm1
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm5, %xmm1, %xmm1
-; AVX512F-NEXT: vpextrd $1, %xmm1, %eax
-; AVX512F-NEXT: vmovd %xmm1, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX512F-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; AVX512F-NEXT: vpaddq %xmm4, %xmm1, %xmm1
+; AVX512F-NEXT: vmovq %xmm1, %rax
+; AVX512F-NEXT: andl $31, %eax
; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm0
; AVX512F-NEXT: vpmovzxwd {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero
; AVX512F-NEXT: vpcompressd %zmm0, %zmm0 {%k1} {z}
-; AVX512F-NEXT: vpmovdw %zmm0, (%rsp,%rcx,2)
+; AVX512F-NEXT: vpmovdw %zmm0, (%rsp,%rax,2)
; AVX512F-NEXT: vextracti64x4 $1, %zmm2, %ymm0
-; AVX512F-NEXT: vpsllw $15, %ymm4, %ymm1
+; AVX512F-NEXT: vpsllw $15, %ymm5, %ymm1
; AVX512F-NEXT: vpsraw $15, %ymm1, %ymm1
; AVX512F-NEXT: vpblendvb %ymm1, {{[0-9]+}}(%rsp), %ymm0, %ymm0
; AVX512F-NEXT: vpsllw $15, %ymm3, %ymm1
@@ -3355,33 +3307,32 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm0 {%k4} {z} = -1
; AVX512F-NEXT: vpsrld $31, %zmm0, %zmm4
; AVX512F-NEXT: vextracti64x4 $1, %zmm4, %ymm5
-; AVX512F-NEXT: vpaddd %ymm5, %ymm4, %ymm4
-; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm5
-; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm5, %xmm4, %xmm4
-; AVX512F-NEXT: vpextrd $1, %xmm4, %eax
-; AVX512F-NEXT: vmovd %xmm4, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vpaddb %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX512F-NEXT: vpsadbw %ymm5, %ymm4, %ymm4
+; AVX512F-NEXT: vextracti128 $1, %ymm4, %xmm6
+; AVX512F-NEXT: vpaddq %xmm6, %xmm4, %xmm4
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm6 = xmm4[2,3,2,3]
+; AVX512F-NEXT: vpaddq %xmm6, %xmm4, %xmm4
+; AVX512F-NEXT: vmovq %xmm4, %rax
+; AVX512F-NEXT: andl $31, %eax
; AVX512F-NEXT: vpcompressd %zmm1, %zmm1 {%k2} {z}
-; AVX512F-NEXT: vmovdqa64 %zmm1, (%rsp,%rcx,4)
+; AVX512F-NEXT: vmovdqa64 %zmm1, (%rsp,%rax,4)
; AVX512F-NEXT: vpcompressd %zmm2, %zmm1 {%k3} {z}
; AVX512F-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
; AVX512F-NEXT: vpternlogd {{.*#+}} zmm1 {%k3} {z} = -1
; AVX512F-NEXT: vpsrld $31, %zmm1, %zmm1
; AVX512F-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512F-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT: vpaddb %ymm2, %ymm1, %ymm1
+; AVX512F-NEXT: vpsadbw %ymm5, %ymm1, %ymm1
; AVX512F-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX512F-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT: vpaddq %xmm2, %xmm1, %xmm1
; AVX512F-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; AVX512F-NEXT: vpaddd %xmm2, %xmm1, %xmm1
-; AVX512F-NEXT: vpextrd $1, %xmm1, %eax
-; AVX512F-NEXT: vmovd %xmm1, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $31, %ecx
+; AVX512F-NEXT: vpaddq %xmm2, %xmm1, %xmm1
+; AVX512F-NEXT: vmovq %xmm1, %rax
+; AVX512F-NEXT: andl $31, %eax
; AVX512F-NEXT: vpcompressd %zmm3, %zmm1 {%k1} {z}
-; AVX512F-NEXT: vmovdqa64 %zmm1, 128(%rsp,%rcx,4)
+; AVX512F-NEXT: vmovdqa64 %zmm1, 128(%rsp,%rax,4)
; AVX512F-NEXT: vmovdqa64 (%rsp), %zmm1
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm2
; AVX512F-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
@@ -3394,15 +3345,15 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512F-NEXT: vmovd %xmm0, %ecx
-; AVX512F-NEXT: addl %eax, %ecx
-; AVX512F-NEXT: andl $63, %ecx
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512F-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vmovd %xmm0, %eax
+; AVX512F-NEXT: andl $63, %eax
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm0
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
-; AVX512F-NEXT: vmovaps %zmm0, 256(%rsp,%rcx,4)
+; AVX512F-NEXT: vmovaps %zmm0, 256(%rsp,%rax,4)
; AVX512F-NEXT: vmovaps %zmm2, {{[0-9]+}}(%rsp)
-; AVX512F-NEXT: vmovaps %zmm1, 320(%rsp,%rcx,4)
+; AVX512F-NEXT: vmovaps %zmm1, 320(%rsp,%rax,4)
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm0
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
; AVX512F-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm2
@@ -3425,35 +3376,34 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512VL-NEXT: vmovdqa64 %zmm0, (%rsp)
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm0 {%k2} {z} = -1
; AVX512VL-NEXT: vpsrld $31, %zmm0, %zmm1
-; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm5
-; AVX512VL-NEXT: vpaddd %ymm5, %ymm1, %ymm1
-; AVX512VL-NEXT: vextracti128 $1, %ymm1, %xmm5
-; AVX512VL-NEXT: vpaddd %xmm5, %xmm1, %xmm1
-; AVX512VL-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
-; AVX512VL-NEXT: vpaddd %xmm5, %xmm1, %xmm1
-; AVX512VL-NEXT: vpextrd $1, %xmm1, %eax
-; AVX512VL-NEXT: vmovd %xmm1, %ecx
-; AVX512VL-NEXT: addl %eax, %ecx
-; AVX512VL-NEXT: andl $31, %ecx
+; AVX512VL-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX512VL-NEXT: vpsadbw %zmm5, %zmm1, %zmm1
+; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm6
+; AVX512VL-NEXT: vpaddq %ymm6, %ymm1, %ymm1
+; AVX512VL-NEXT: vextracti128 $1, %ymm1, %xmm6
+; AVX512VL-NEXT: vpaddq %xmm6, %xmm1, %xmm1
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm6, %xmm1, %xmm1
+; AVX512VL-NEXT: vmovq %xmm1, %rax
+; AVX512VL-NEXT: andl $31, %eax
; AVX512VL-NEXT: kshiftrq $16, %k2, %k2
; AVX512VL-NEXT: vpcompressd %zmm2, %zmm1 {%k2} {z}
-; AVX512VL-NEXT: vmovdqa64 %zmm1, (%rsp,%rcx,4)
+; AVX512VL-NEXT: vmovdqa64 %zmm1, (%rsp,%rax,4)
; AVX512VL-NEXT: vpcompressd %zmm3, %zmm1 {%k3} {z}
; AVX512VL-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
; AVX512VL-NEXT: vpternlogd {{.*#+}} zmm1 {%k3} {z} = -1
; AVX512VL-NEXT: vpsrld $31, %zmm1, %zmm1
+; AVX512VL-NEXT: vpsadbw %zmm5, %zmm1, %zmm1
; AVX512VL-NEXT: vextracti64x4 $1, %zmm1, %ymm2
-; AVX512VL-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX512VL-NEXT: vpaddq %ymm2, %ymm1, %ymm1
; AVX512VL-NEXT: vextracti128 $1, %ymm1, %xmm2
-; AVX512VL-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT: vpaddq %xmm2, %xmm1, %xmm1
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
-; AVX512VL-NEXT: vpaddd %xmm2, %xmm1, %xmm1
-; AVX512VL-NEXT: vpextrd $1, %xmm1, %eax
-; AVX512VL-NEXT: vmovd %xmm1, %ecx
-; AVX512VL-NEXT: addl %eax, %ecx
-; AVX512VL-NEXT: andl $31, %ecx
+; AVX512VL-NEXT: vpaddq %xmm2, %xmm1, %xmm1
+; AVX512VL-NEXT: vmovq %xmm1, %rax
+; AVX512VL-NEXT: andl $31, %eax
; AVX512VL-NEXT: vpcompressd %zmm4, %zmm1 {%k1} {z}
-; AVX512VL-NEXT: vmovdqa64 %zmm1, 128(%rsp,%rcx,4)
+; AVX512VL-NEXT: vmovdqa64 %zmm1, 128(%rsp,%rax,4)
; AVX512VL-NEXT: vmovdqa64 (%rsp), %zmm1
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm2
; AVX512VL-NEXT: vmovdqa64 %zmm1, {{[0-9]+}}(%rsp)
@@ -3466,15 +3416,15 @@ define <64 x i32> @test_compress_large(<64 x i1> %mask, <64 x i32> %vec, <64 x i
; AVX512VL-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VL-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vpextrd $1, %xmm0, %eax
-; AVX512VL-NEXT: vmovd %xmm0, %ecx
-; AVX512VL-NEXT: addl %eax, %ecx
-; AVX512VL-NEXT: andl $63, %ecx
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512VL-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovd %xmm0, %eax
+; AVX512VL-NEXT: andl $63, %eax
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm0
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
-; AVX512VL-NEXT: vmovaps %zmm0, 256(%rsp,%rcx,4)
+; AVX512VL-NEXT: vmovaps %zmm0, 256(%rsp,%rax,4)
; AVX512VL-NEXT: vmovaps %zmm2, {{[0-9]+}}(%rsp)
-; AVX512VL-NEXT: vmovaps %zmm1, 320(%rsp,%rcx,4)
+; AVX512VL-NEXT: vmovaps %zmm1, 320(%rsp,%rax,4)
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm0
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm1
; AVX512VL-NEXT: vmovaps {{[0-9]+}}(%rsp), %zmm2
@@ -3509,9 +3459,9 @@ define <8 x i64> @test_compress_knownbits_zext_v8i16_8i64(<8 x i16> %vec, <8 x
; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vpextrw $1, %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
@@ -3613,9 +3563,9 @@ define <8 x i64> @test_compress_knownbits_sext_v8i16_8i64(<8 x i16> %vec, <8 x i
; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm2
; AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
-; AVX2-NEXT: vpextrq $1, %xmm2, %rcx
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
; AVX2-NEXT: vmovq %xmm2, %rax
-; AVX2-NEXT: addl %ecx, %eax
; AVX2-NEXT: andl $7, %eax
; AVX2-NEXT: vpextrw $1, %xmm1, %ecx
; AVX2-NEXT: andl $1, %ecx
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll b/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
index 43bc6f5ef8429..6f77d3fcb5c72 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-mask.ll
@@ -180,27 +180,16 @@ define i64 @test_v4i64_v4i16(<4 x i64> %a0) nounwind {
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
-; AVX512BW-LABEL: test_v4i64_v4i16:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512BW-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vmovq %xmm0, %rax
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
-;
-; AVX512BWVL-LABEL: test_v4i64_v4i16:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512BWVL-NEXT: vpmovqb %ymm0, %xmm0
-; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vmovq %xmm0, %rax
-; AVX512BWVL-NEXT: vzeroupper
-; AVX512BWVL-NEXT: retq
+; AVX512-LABEL: test_v4i64_v4i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%1 = and <4 x i64> %a0, <i64 15, i64 31, i64 63, i64 127>
%2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %1)
ret i64 %2
@@ -331,9 +320,12 @@ define i64 @test_v8i64_v8i8(<8 x i64> %a0) nounwind {
; AVX512-LABEL: test_v8i64_v8i8:
; AVX512: # %bb.0:
; AVX512-NEXT: vpsrlq $60, %zmm0, %zmm0
-; AVX512-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovq %xmm0, %rax
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
@@ -350,25 +342,25 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [1,0,1,0]
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
; X86-SSE2-NEXT: pand %xmm3, %xmm0
; X86-SSE2-NEXT: pand %xmm3, %xmm2
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm4
-; X86-SSE2-NEXT: pand %xmm3, %xmm4
-; X86-SSE2-NEXT: paddq %xmm1, %xmm4
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm5
-; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm1
; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: paddq %xmm5, %xmm1
-; X86-SSE2-NEXT: paddq %xmm4, %xmm1
; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm4
; X86-SSE2-NEXT: pand %xmm3, %xmm4
; X86-SSE2-NEXT: paddq %xmm0, %xmm4
-; X86-SSE2-NEXT: pand 56(%ebp), %xmm3
-; X86-SSE2-NEXT: paddq %xmm2, %xmm3
-; X86-SSE2-NEXT: paddq %xmm4, %xmm3
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm0
+; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: paddq %xmm2, %xmm0
+; X86-SSE2-NEXT: paddq %xmm4, %xmm0
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm2
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: paddq %xmm1, %xmm2
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE2-NEXT: pand %xmm3, %xmm1
+; X86-SSE2-NEXT: pand 8(%ebp), %xmm3
; X86-SSE2-NEXT: paddq %xmm1, %xmm3
+; X86-SSE2-NEXT: paddq %xmm2, %xmm3
+; X86-SSE2-NEXT: paddq %xmm0, %xmm3
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
; X86-SSE2-NEXT: paddq %xmm3, %xmm0
; X86-SSE2-NEXT: movd %xmm0, %eax
@@ -381,13 +373,6 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X64-SSE2-LABEL: test_v16i64_v16i8:
; X64-SSE2: # %bb.0:
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm8 = [1,1]
-; X64-SSE2-NEXT: pand %xmm8, %xmm5
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
-; X64-SSE2-NEXT: paddq %xmm5, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm7
-; X64-SSE2-NEXT: pand %xmm8, %xmm3
-; X64-SSE2-NEXT: paddq %xmm7, %xmm3
-; X64-SSE2-NEXT: paddq %xmm1, %xmm3
; X64-SSE2-NEXT: pand %xmm8, %xmm4
; X64-SSE2-NEXT: pand %xmm8, %xmm0
; X64-SSE2-NEXT: paddq %xmm4, %xmm0
@@ -395,9 +380,16 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X64-SSE2-NEXT: pand %xmm8, %xmm2
; X64-SSE2-NEXT: paddq %xmm6, %xmm2
; X64-SSE2-NEXT: paddq %xmm0, %xmm2
-; X64-SSE2-NEXT: paddq %xmm3, %xmm2
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm2, %xmm0
+; X64-SSE2-NEXT: pand %xmm8, %xmm5
+; X64-SSE2-NEXT: pand %xmm8, %xmm1
+; X64-SSE2-NEXT: paddq %xmm5, %xmm1
+; X64-SSE2-NEXT: pand %xmm8, %xmm7
+; X64-SSE2-NEXT: pand %xmm8, %xmm3
+; X64-SSE2-NEXT: paddq %xmm7, %xmm3
+; X64-SSE2-NEXT: paddq %xmm1, %xmm3
+; X64-SSE2-NEXT: paddq %xmm2, %xmm3
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X64-SSE2-NEXT: paddq %xmm3, %xmm0
; X64-SSE2-NEXT: movq %xmm0, %rax
; X64-SSE2-NEXT: retq
;
@@ -408,25 +400,25 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
; X86-SSE4-NEXT: pmovsxbq {{.*#+}} xmm3 = [1,1]
-; X86-SSE4-NEXT: pand %xmm3, %xmm1
; X86-SSE4-NEXT: pand %xmm3, %xmm0
; X86-SSE4-NEXT: pand %xmm3, %xmm2
-; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm4
-; X86-SSE4-NEXT: pand %xmm3, %xmm4
-; X86-SSE4-NEXT: paddq %xmm1, %xmm4
-; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm5
-; X86-SSE4-NEXT: pand %xmm3, %xmm5
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm1
; X86-SSE4-NEXT: pand %xmm3, %xmm1
-; X86-SSE4-NEXT: paddq %xmm5, %xmm1
-; X86-SSE4-NEXT: paddq %xmm4, %xmm1
; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm4
; X86-SSE4-NEXT: pand %xmm3, %xmm4
; X86-SSE4-NEXT: paddq %xmm0, %xmm4
-; X86-SSE4-NEXT: pand 56(%ebp), %xmm3
-; X86-SSE4-NEXT: paddq %xmm2, %xmm3
-; X86-SSE4-NEXT: paddq %xmm4, %xmm3
+; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm0
+; X86-SSE4-NEXT: pand %xmm3, %xmm0
+; X86-SSE4-NEXT: paddq %xmm2, %xmm0
+; X86-SSE4-NEXT: paddq %xmm4, %xmm0
+; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm2
+; X86-SSE4-NEXT: pand %xmm3, %xmm2
+; X86-SSE4-NEXT: paddq %xmm1, %xmm2
+; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE4-NEXT: pand %xmm3, %xmm1
+; X86-SSE4-NEXT: pand 8(%ebp), %xmm3
; X86-SSE4-NEXT: paddq %xmm1, %xmm3
+; X86-SSE4-NEXT: paddq %xmm2, %xmm3
+; X86-SSE4-NEXT: paddq %xmm0, %xmm3
; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
; X86-SSE4-NEXT: paddq %xmm3, %xmm0
; X86-SSE4-NEXT: movd %xmm0, %eax
@@ -438,13 +430,6 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X64-SSE4-LABEL: test_v16i64_v16i8:
; X64-SSE4: # %bb.0:
; X64-SSE4-NEXT: pmovsxbq {{.*#+}} xmm8 = [1,1]
-; X64-SSE4-NEXT: pand %xmm8, %xmm5
-; X64-SSE4-NEXT: pand %xmm8, %xmm1
-; X64-SSE4-NEXT: paddq %xmm5, %xmm1
-; X64-SSE4-NEXT: pand %xmm8, %xmm7
-; X64-SSE4-NEXT: pand %xmm8, %xmm3
-; X64-SSE4-NEXT: paddq %xmm7, %xmm3
-; X64-SSE4-NEXT: paddq %xmm1, %xmm3
; X64-SSE4-NEXT: pand %xmm8, %xmm4
; X64-SSE4-NEXT: pand %xmm8, %xmm0
; X64-SSE4-NEXT: paddq %xmm4, %xmm0
@@ -452,9 +437,16 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X64-SSE4-NEXT: pand %xmm8, %xmm2
; X64-SSE4-NEXT: paddq %xmm6, %xmm2
; X64-SSE4-NEXT: paddq %xmm0, %xmm2
-; X64-SSE4-NEXT: paddq %xmm3, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm2, %xmm0
+; X64-SSE4-NEXT: pand %xmm8, %xmm5
+; X64-SSE4-NEXT: pand %xmm8, %xmm1
+; X64-SSE4-NEXT: paddq %xmm5, %xmm1
+; X64-SSE4-NEXT: pand %xmm8, %xmm7
+; X64-SSE4-NEXT: pand %xmm8, %xmm3
+; X64-SSE4-NEXT: paddq %xmm7, %xmm3
+; X64-SSE4-NEXT: paddq %xmm1, %xmm3
+; X64-SSE4-NEXT: paddq %xmm2, %xmm3
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm3, %xmm0
; X64-SSE4-NEXT: movq %xmm0, %rax
; X64-SSE4-NEXT: retq
;
@@ -469,17 +461,17 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X86-AVX1-NEXT: vandps %ymm3, %ymm0, %ymm0
; X86-AVX1-NEXT: vandps %ymm3, %ymm2, %ymm2
; X86-AVX1-NEXT: vandps 8(%ebp), %ymm3, %ymm3
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm4
-; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5
+; X86-AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm6
+; X86-AVX1-NEXT: vpaddq %xmm6, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpaddq %xmm5, %xmm4, %xmm4
; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm2
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
; X86-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
+; X86-AVX1-NEXT: vpaddq %xmm4, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
@@ -496,17 +488,17 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X64-AVX1-NEXT: vandps %ymm4, %ymm2, %ymm2
; X64-AVX1-NEXT: vandps %ymm4, %ymm1, %ymm1
; X64-AVX1-NEXT: vandps %ymm4, %ymm3, %ymm3
-; X64-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm4
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-NEXT: vpaddq %xmm5, %xmm6, %xmm5
; X64-AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm0, %xmm4, %xmm0
+; X64-AVX1-NEXT: vpaddq %xmm4, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
@@ -556,33 +548,21 @@ define i64 @test_v16i64_v16i8(<16 x i64> %a0) nounwind {
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
-; AVX512BW-LABEL: test_v16i64_v16i8:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpmovqb %zmm1, %xmm1
-; AVX512BW-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512BW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BW-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vmovq %xmm0, %rax
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
-;
-; AVX512BWVL-LABEL: test_v16i64_v16i8:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpmovqb %zmm1, %xmm1
-; AVX512BWVL-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512BWVL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512BWVL-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to4}, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BWVL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vmovq %xmm0, %rax
-; AVX512BWVL-NEXT: vzeroupper
-; AVX512BWVL-NEXT: retq
+; AVX512-LABEL: test_v16i64_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpbroadcastq {{.*#+}} zmm2 = [1,1,1,1,1,1,1,1]
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm1
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%1 = and <16 x i64> %a0, <i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1, i64 1>
%2 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %1)
ret i64 %2
@@ -596,71 +576,61 @@ define i32 @test_v2i32_v2i16(<2 x i32> %a0) nounwind {
; X86-SSE-LABEL: test_v2i32_v2i16:
; X86-SSE: # %bb.0:
; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-SSE-NEXT: paddd %xmm0, %xmm1
+; X86-SSE-NEXT: pxor %xmm1, %xmm1
+; X86-SSE-NEXT: psadbw %xmm0, %xmm1
; X86-SSE-NEXT: movd %xmm1, %eax
; X86-SSE-NEXT: retl
;
; X64-SSE-LABEL: test_v2i32_v2i16:
; X64-SSE: # %bb.0:
; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-SSE-NEXT: paddd %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
+; X64-SSE-NEXT: pxor %xmm1, %xmm1
+; X64-SSE-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE-NEXT: movq %xmm1, %rax
+; X64-SSE-NEXT: # kill: def $eax killed $eax killed $rax
; X64-SSE-NEXT: retq
;
-; X86-AVX1-SLOW-LABEL: test_v2i32_v2i16:
-; X86-AVX1-SLOW: # %bb.0:
-; X86-AVX1-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-SLOW-NEXT: retl
-;
-; X64-AVX1-SLOW-LABEL: test_v2i32_v2i16:
-; X64-AVX1-SLOW: # %bb.0:
-; X64-AVX1-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-SLOW-NEXT: retq
-;
-; X86-AVX1-FAST-LABEL: test_v2i32_v2i16:
-; X86-AVX1-FAST: # %bb.0:
-; X86-AVX1-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-FAST-NEXT: retl
+; X86-AVX1-LABEL: test_v2i32_v2i16:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: retl
;
-; X64-AVX1-FAST-LABEL: test_v2i32_v2i16:
-; X64-AVX1-FAST: # %bb.0:
-; X64-AVX1-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-FAST-NEXT: retq
+; X64-AVX1-LABEL: test_v2i32_v2i16:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: test_v2i32_v2i16:
; X86-AVX2: # %bb.0:
; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: retl
;
; X64-AVX2-LABEL: test_v2i32_v2i16:
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vmovd %xmm0, %eax
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; X64-AVX2-NEXT: retq
;
; AVX512-LABEL: test_v2i32_v2i16:
; AVX512: # %bb.0:
; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%1 = and <2 x i32> %a0, <i32 255, i32 255>
%2 = call i32 @llvm.vector.reduce.add.v2i32(<2 x i32> %1)
@@ -668,61 +638,79 @@ define i32 @test_v2i32_v2i16(<2 x i32> %a0) nounwind {
}
define i32 @test_v4i32(<4 x i32> %a0) nounwind {
-; SSE-LABEL: test_v4i32:
-; SSE: # %bb.0:
-; SSE-NEXT: psrld $31, %xmm0
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE-NEXT: paddd %xmm0, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; SSE-NEXT: paddd %xmm1, %xmm0
-; SSE-NEXT: movd %xmm0, %eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX1-SLOW-LABEL: test_v4i32:
-; AVX1-SLOW: # %bb.0:
-; AVX1-SLOW-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; AVX1-SLOW-NEXT: ret{{[l|q]}}
+; X86-SSE-LABEL: test_v4i32:
+; X86-SSE: # %bb.0:
+; X86-SSE-NEXT: psrld $31, %xmm0
+; X86-SSE-NEXT: pxor %xmm1, %xmm1
+; X86-SSE-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE-NEXT: paddq %xmm1, %xmm0
+; X86-SSE-NEXT: movd %xmm0, %eax
+; X86-SSE-NEXT: retl
;
-; AVX1-FAST-LABEL: test_v4i32:
-; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; AVX1-FAST-NEXT: ret{{[l|q]}}
+; X64-SSE-LABEL: test_v4i32:
+; X64-SSE: # %bb.0:
+; X64-SSE-NEXT: psrld $31, %xmm0
+; X64-SSE-NEXT: pxor %xmm1, %xmm1
+; X64-SSE-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddq %xmm1, %xmm0
+; X64-SSE-NEXT: movq %xmm0, %rax
+; X64-SSE-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-SSE-NEXT: retq
;
-; AVX2-LABEL: test_v4i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: ret{{[l|q]}}
+; X86-AVX1-LABEL: test_v4i32:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpsrld $31, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: retl
;
-; AVX512BW-LABEL: test_v4i32:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX512BW-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,4,8,12],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vmovd %xmm0, %eax
-; AVX512BW-NEXT: retq
+; X64-AVX1-LABEL: test_v4i32:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpsrld $31, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX1-NEXT: retq
;
-; AVX512BWVL-LABEL: test_v4i32:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpsrld $31, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpmovdb %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vmovd %xmm0, %eax
-; AVX512BWVL-NEXT: retq
+; X86-AVX2-LABEL: test_v4i32:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpsrld $31, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: test_v4i32:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpsrld $31, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX2-NEXT: retq
+;
+; AVX512-LABEL: test_v4i32:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpsrld $31, %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX512-NEXT: retq
%1 = lshr <4 x i32> %a0, <i32 31, i32 31, i32 31, i32 31>
%2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)
ret i32 %2
@@ -734,10 +722,10 @@ define i32 @test_v8i32_v8i8(<8 x i32> %a0) nounwind {
; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm1
; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
; X86-SSE-NEXT: por %xmm1, %xmm0
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE-NEXT: paddd %xmm0, %xmm1
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X86-SSE-NEXT: paddd %xmm1, %xmm0
+; X86-SSE-NEXT: pxor %xmm1, %xmm1
+; X86-SSE-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE-NEXT: paddq %xmm1, %xmm0
; X86-SSE-NEXT: movd %xmm0, %eax
; X86-SSE-NEXT: retl
;
@@ -746,70 +734,50 @@ define i32 @test_v8i32_v8i8(<8 x i32> %a0) nounwind {
; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
; X64-SSE-NEXT: por %xmm1, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE-NEXT: paddd %xmm0, %xmm1
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X64-SSE-NEXT: paddd %xmm1, %xmm0
-; X64-SSE-NEXT: movd %xmm0, %eax
+; X64-SSE-NEXT: pxor %xmm1, %xmm1
+; X64-SSE-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddq %xmm1, %xmm0
+; X64-SSE-NEXT: movq %xmm0, %rax
+; X64-SSE-NEXT: # kill: def $eax killed $eax killed $rax
; X64-SSE-NEXT: retq
;
-; X86-AVX1-SLOW-LABEL: test_v8i32_v8i8:
-; X86-AVX1-SLOW: # %bb.0:
-; X86-AVX1-SLOW-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X86-AVX1-SLOW-NEXT: vorps %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-SLOW-NEXT: vzeroupper
-; X86-AVX1-SLOW-NEXT: retl
+; X86-AVX1-LABEL: test_v8i32_v8i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X86-AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: vzeroupper
+; X86-AVX1-NEXT: retl
;
-; X64-AVX1-SLOW-LABEL: test_v8i32_v8i8:
-; X64-AVX1-SLOW: # %bb.0:
-; X64-AVX1-SLOW-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X64-AVX1-SLOW-NEXT: vorps %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-SLOW-NEXT: vzeroupper
-; X64-AVX1-SLOW-NEXT: retq
-;
-; X86-AVX1-FAST-LABEL: test_v8i32_v8i8:
-; X86-AVX1-FAST: # %bb.0:
-; X86-AVX1-FAST-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
-; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-FAST-NEXT: vzeroupper
-; X86-AVX1-FAST-NEXT: retl
-;
-; X64-AVX1-FAST-LABEL: test_v8i32_v8i8:
-; X64-AVX1-FAST: # %bb.0:
-; X64-AVX1-FAST-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
-; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-FAST-NEXT: vzeroupper
-; X64-AVX1-FAST-NEXT: retq
+; X64-AVX1-LABEL: test_v8i32_v8i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
+; X64-AVX1-NEXT: vorps %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX1-NEXT: vzeroupper
+; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: test_v8i32_v8i8:
; X86-AVX2: # %bb.0:
; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X86-AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: vzeroupper
; X86-AVX2-NEXT: retl
@@ -817,35 +785,30 @@ define i32 @test_v8i32_v8i8(<8 x i32> %a0) nounwind {
; X64-AVX2-LABEL: test_v8i32_v8i8:
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X64-AVX2-NEXT: vpor %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vmovd %xmm0, %eax
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $eax killed $eax killed $rax
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
-; AVX512BW-LABEL: test_v8i32_v8i8:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512BW-NEXT: vpmovdb %zmm0, %xmm0
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vmovd %xmm0, %eax
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
-;
-; AVX512BWVL-LABEL: test_v8i32_v8i8:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; AVX512BWVL-NEXT: vpmovdb %ymm0, %xmm0
-; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vmovd %xmm0, %eax
-; AVX512BWVL-NEXT: vzeroupper
-; AVX512BWVL-NEXT: retq
+; AVX512-LABEL: test_v8i32_v8i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%1 = and <8 x i32> %a0, <i32 0, i32 1, i32 2, i32 4, i32 8, i32 16, i32 32, i32 64>
%2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
ret i32 %2
@@ -960,8 +923,7 @@ define i32 @test_v16i32_v16i8(<16 x i32> %a0) nounwind {
; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: vzeroupper
@@ -985,12 +947,17 @@ define i32 @test_v16i32_v16i8(<16 x i32> %a0) nounwind {
;
; AVX512-LABEL: test_v16i32_v16i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpmovdb %zmm0, %xmm0
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%1 = and <16 x i32> %a0, <i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255, i32 255>
@@ -1006,25 +973,25 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
; X86-SSE2-NEXT: pand %xmm3, %xmm0
; X86-SSE2-NEXT: pand %xmm3, %xmm2
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm4
-; X86-SSE2-NEXT: pand %xmm3, %xmm4
-; X86-SSE2-NEXT: paddd %xmm1, %xmm4
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm5
-; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm1
; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: paddd %xmm5, %xmm1
-; X86-SSE2-NEXT: paddd %xmm4, %xmm1
; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm4
; X86-SSE2-NEXT: pand %xmm3, %xmm4
; X86-SSE2-NEXT: paddd %xmm0, %xmm4
-; X86-SSE2-NEXT: pand 56(%ebp), %xmm3
-; X86-SSE2-NEXT: paddd %xmm2, %xmm3
-; X86-SSE2-NEXT: paddd %xmm4, %xmm3
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm0
+; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: paddd %xmm2, %xmm0
+; X86-SSE2-NEXT: paddd %xmm4, %xmm0
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm2
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: paddd %xmm1, %xmm2
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE2-NEXT: pand %xmm3, %xmm1
+; X86-SSE2-NEXT: pand 8(%ebp), %xmm3
; X86-SSE2-NEXT: paddd %xmm1, %xmm3
+; X86-SSE2-NEXT: paddd %xmm2, %xmm3
+; X86-SSE2-NEXT: paddd %xmm0, %xmm3
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
; X86-SSE2-NEXT: paddd %xmm3, %xmm0
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1037,13 +1004,6 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X64-SSE2-LABEL: test_v32i32_v32i8:
; X64-SSE2: # %bb.0:
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm8 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
-; X64-SSE2-NEXT: pand %xmm8, %xmm5
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
-; X64-SSE2-NEXT: paddd %xmm5, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm7
-; X64-SSE2-NEXT: pand %xmm8, %xmm3
-; X64-SSE2-NEXT: paddd %xmm7, %xmm3
-; X64-SSE2-NEXT: paddd %xmm1, %xmm3
; X64-SSE2-NEXT: pand %xmm8, %xmm4
; X64-SSE2-NEXT: pand %xmm8, %xmm0
; X64-SSE2-NEXT: paddd %xmm4, %xmm0
@@ -1051,9 +1011,16 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X64-SSE2-NEXT: pand %xmm8, %xmm2
; X64-SSE2-NEXT: paddd %xmm6, %xmm2
; X64-SSE2-NEXT: paddd %xmm0, %xmm2
-; X64-SSE2-NEXT: paddd %xmm3, %xmm2
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; X64-SSE2-NEXT: paddd %xmm2, %xmm0
+; X64-SSE2-NEXT: pand %xmm8, %xmm5
+; X64-SSE2-NEXT: pand %xmm8, %xmm1
+; X64-SSE2-NEXT: paddd %xmm5, %xmm1
+; X64-SSE2-NEXT: pand %xmm8, %xmm7
+; X64-SSE2-NEXT: pand %xmm8, %xmm3
+; X64-SSE2-NEXT: paddd %xmm7, %xmm3
+; X64-SSE2-NEXT: paddd %xmm1, %xmm3
+; X64-SSE2-NEXT: paddd %xmm2, %xmm3
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X64-SSE2-NEXT: paddd %xmm3, %xmm0
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; X64-SSE2-NEXT: paddd %xmm0, %xmm1
; X64-SSE2-NEXT: movd %xmm1, %eax
@@ -1066,25 +1033,25 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm3 = [255,255,255,255]
-; X86-SSE4-NEXT: pand %xmm3, %xmm1
; X86-SSE4-NEXT: pand %xmm3, %xmm0
; X86-SSE4-NEXT: pand %xmm3, %xmm2
-; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm4
-; X86-SSE4-NEXT: pand %xmm3, %xmm4
-; X86-SSE4-NEXT: paddd %xmm1, %xmm4
-; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm5
-; X86-SSE4-NEXT: pand %xmm3, %xmm5
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm1
; X86-SSE4-NEXT: pand %xmm3, %xmm1
-; X86-SSE4-NEXT: paddd %xmm5, %xmm1
-; X86-SSE4-NEXT: paddd %xmm4, %xmm1
; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm4
; X86-SSE4-NEXT: pand %xmm3, %xmm4
; X86-SSE4-NEXT: paddd %xmm0, %xmm4
-; X86-SSE4-NEXT: pand 56(%ebp), %xmm3
-; X86-SSE4-NEXT: paddd %xmm2, %xmm3
-; X86-SSE4-NEXT: paddd %xmm4, %xmm3
+; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm0
+; X86-SSE4-NEXT: pand %xmm3, %xmm0
+; X86-SSE4-NEXT: paddd %xmm2, %xmm0
+; X86-SSE4-NEXT: paddd %xmm4, %xmm0
+; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm2
+; X86-SSE4-NEXT: pand %xmm3, %xmm2
+; X86-SSE4-NEXT: paddd %xmm1, %xmm2
+; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE4-NEXT: pand %xmm3, %xmm1
+; X86-SSE4-NEXT: pand 8(%ebp), %xmm3
; X86-SSE4-NEXT: paddd %xmm1, %xmm3
+; X86-SSE4-NEXT: paddd %xmm2, %xmm3
+; X86-SSE4-NEXT: paddd %xmm0, %xmm3
; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
; X86-SSE4-NEXT: paddd %xmm3, %xmm0
; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1097,13 +1064,6 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X64-SSE4-LABEL: test_v32i32_v32i8:
; X64-SSE4: # %bb.0:
; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm8 = [255,255,255,255]
-; X64-SSE4-NEXT: pand %xmm8, %xmm5
-; X64-SSE4-NEXT: pand %xmm8, %xmm1
-; X64-SSE4-NEXT: paddd %xmm5, %xmm1
-; X64-SSE4-NEXT: pand %xmm8, %xmm7
-; X64-SSE4-NEXT: pand %xmm8, %xmm3
-; X64-SSE4-NEXT: paddd %xmm7, %xmm3
-; X64-SSE4-NEXT: paddd %xmm1, %xmm3
; X64-SSE4-NEXT: pand %xmm8, %xmm4
; X64-SSE4-NEXT: pand %xmm8, %xmm0
; X64-SSE4-NEXT: paddd %xmm4, %xmm0
@@ -1111,9 +1071,16 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X64-SSE4-NEXT: pand %xmm8, %xmm2
; X64-SSE4-NEXT: paddd %xmm6, %xmm2
; X64-SSE4-NEXT: paddd %xmm0, %xmm2
-; X64-SSE4-NEXT: paddd %xmm3, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
-; X64-SSE4-NEXT: paddd %xmm2, %xmm0
+; X64-SSE4-NEXT: pand %xmm8, %xmm5
+; X64-SSE4-NEXT: pand %xmm8, %xmm1
+; X64-SSE4-NEXT: paddd %xmm5, %xmm1
+; X64-SSE4-NEXT: pand %xmm8, %xmm7
+; X64-SSE4-NEXT: pand %xmm8, %xmm3
+; X64-SSE4-NEXT: paddd %xmm7, %xmm3
+; X64-SSE4-NEXT: paddd %xmm1, %xmm3
+; X64-SSE4-NEXT: paddd %xmm2, %xmm3
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X64-SSE4-NEXT: paddd %xmm3, %xmm0
; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; X64-SSE4-NEXT: paddd %xmm0, %xmm1
; X64-SSE4-NEXT: movd %xmm1, %eax
@@ -1130,17 +1097,17 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X86-AVX1-SLOW-NEXT: vandps %ymm3, %ymm0, %ymm0
; X86-AVX1-SLOW-NEXT: vandps %ymm3, %ymm2, %ymm2
; X86-AVX1-SLOW-NEXT: vandps 8(%ebp), %ymm3, %ymm3
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm4
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm4
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm5
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm6
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm6, %xmm5, %xmm5
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm4, %xmm4
; X86-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm1, %xmm2
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm4, %xmm2
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm3
; X86-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm1, %xmm1
; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm2, %xmm0
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1158,17 +1125,17 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm2, %ymm2
; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm1, %ymm1
; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm3, %ymm3
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm1, %xmm4
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm6, %xmm5
; X64-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm5, %xmm4
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm1, %xmm1
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm4, %xmm0
+; X64-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1188,19 +1155,18 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X86-AVX1-FAST-NEXT: vandps %ymm3, %ymm0, %ymm0
; X86-AVX1-FAST-NEXT: vandps %ymm3, %ymm2, %ymm2
; X86-AVX1-FAST-NEXT: vandps 8(%ebp), %ymm3, %ymm3
-; X86-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm4
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm4
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm5
+; X86-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm6
+; X86-AVX1-FAST-NEXT: vpaddd %xmm6, %xmm5, %xmm5
+; X86-AVX1-FAST-NEXT: vpaddd %xmm5, %xmm4, %xmm4
; X86-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm2
-; X86-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm4, %xmm2
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm3
; X86-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm1
; X86-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpaddd %xmm0, %xmm2, %xmm0
-; X86-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X86-AVX1-FAST-NEXT: movl %ebp, %esp
@@ -1215,19 +1181,18 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm2, %ymm2
; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm1, %ymm1
; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm3, %ymm3
-; X64-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm4
-; X64-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-FAST-NEXT: vpaddd %xmm5, %xmm6, %xmm5
; X64-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm5, %xmm4
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm1
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vpaddd %xmm0, %xmm4, %xmm0
-; X64-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X64-AVX1-FAST-NEXT: vzeroupper
@@ -1281,15 +1246,18 @@ define i32 @test_v32i32_v32i8(<32 x i32> %a0) nounwind {
;
; AVX512-LABEL: test_v32i32_v32i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpmovdb %zmm0, %xmm0
-; AVX512-NEXT: vpmovdb %zmm1, %xmm1
-; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpbroadcastd {{.*#+}} zmm2 = [255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0,255,0,0,0]
+; AVX512-NEXT: vpandd %zmm2, %zmm1, %zmm1
+; AVX512-NEXT: vpandd %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
@@ -1306,9 +1274,8 @@ define i16 @test_v2i16_v2i8(<2 x i16> %a0) nounwind {
; X86-SSE-LABEL: test_v2i16_v2i8:
; X86-SSE: # %bb.0:
; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: movdqa %xmm0, %xmm1
-; X86-SSE-NEXT: psrld $16, %xmm1
-; X86-SSE-NEXT: paddw %xmm0, %xmm1
+; X86-SSE-NEXT: pxor %xmm1, %xmm1
+; X86-SSE-NEXT: psadbw %xmm0, %xmm1
; X86-SSE-NEXT: movd %xmm1, %eax
; X86-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE-NEXT: retl
@@ -1316,52 +1283,35 @@ define i16 @test_v2i16_v2i8(<2 x i16> %a0) nounwind {
; X64-SSE-LABEL: test_v2i16_v2i8:
; X64-SSE: # %bb.0:
; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; X64-SSE-NEXT: movdqa %xmm0, %xmm1
-; X64-SSE-NEXT: psrld $16, %xmm1
-; X64-SSE-NEXT: paddw %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
-; X64-SSE-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE-NEXT: pxor %xmm1, %xmm1
+; X64-SSE-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE-NEXT: movq %xmm1, %rax
+; X64-SSE-NEXT: # kill: def $ax killed $ax killed $rax
; X64-SSE-NEXT: retq
;
-; X86-AVX1-SLOW-LABEL: test_v2i16_v2i8:
-; X86-AVX1-SLOW: # %bb.0:
-; X86-AVX1-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
-; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-AVX1-SLOW-NEXT: retl
-;
-; X64-AVX1-SLOW-LABEL: test_v2i16_v2i8:
-; X64-AVX1-SLOW: # %bb.0:
-; X64-AVX1-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
-; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-AVX1-SLOW-NEXT: retq
-;
-; X86-AVX1-FAST-LABEL: test_v2i16_v2i8:
-; X86-AVX1-FAST: # %bb.0:
-; X86-AVX1-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-AVX1-FAST-NEXT: retl
+; X86-AVX1-LABEL: test_v2i16_v2i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-NEXT: retl
;
-; X64-AVX1-FAST-LABEL: test_v2i16_v2i8:
-; X64-AVX1-FAST: # %bb.0:
-; X64-AVX1-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-AVX1-FAST-NEXT: retq
+; X64-AVX1-LABEL: test_v2i16_v2i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: test_v2i16_v2i8:
; X86-AVX2: # %bb.0:
; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
-; X86-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; X86-AVX2-NEXT: retl
@@ -1369,19 +1319,19 @@ define i16 @test_v2i16_v2i8(<2 x i16> %a0) nounwind {
; X64-AVX2-LABEL: test_v2i16_v2i8:
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
-; X64-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vmovd %xmm0, %eax
-; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $rax
; X64-AVX2-NEXT: retq
;
; AVX512-LABEL: test_v2i16_v2i8:
; AVX512: # %bb.0:
; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $rax
; AVX512-NEXT: retq
%1 = and <2 x i16> %a0, <i16 255, i16 255>
%2 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %1)
@@ -1461,8 +1411,7 @@ define i16 @test_v4i16_v4i8(<4 x i16> %a0) nounwind {
; X86-AVX1-FAST: # %bb.0:
; X86-AVX1-FAST-NEXT: vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm1 # [u,32768,16384,8192,u,u,u,u]
; X86-AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; X86-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X86-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1472,8 +1421,7 @@ define i16 @test_v4i16_v4i8(<4 x i16> %a0) nounwind {
; X64-AVX1-FAST: # %bb.0:
; X64-AVX1-FAST-NEXT: vpmulhuw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 # [u,32768,16384,8192,u,u,u,u]
; X64-AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
-; X64-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X64-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1536,29 +1484,32 @@ define i16 @test_v8i16_v8i8(<8 x i16> %a0) nounwind {
; X86-SSE-LABEL: test_v8i16_v8i8:
; X86-SSE: # %bb.0:
; X86-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
-; X86-SSE-NEXT: packuswb %xmm0, %xmm0
; X86-SSE-NEXT: pxor %xmm1, %xmm1
; X86-SSE-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE-NEXT: movd %xmm1, %eax
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE-NEXT: paddq %xmm1, %xmm0
+; X86-SSE-NEXT: movd %xmm0, %eax
; X86-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE-NEXT: retl
;
; X64-SSE-LABEL: test_v8i16_v8i8:
; X64-SSE: # %bb.0:
; X64-SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
-; X64-SSE-NEXT: packuswb %xmm0, %xmm0
; X64-SSE-NEXT: pxor %xmm1, %xmm1
; X64-SSE-NEXT: psadbw %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
-; X64-SSE-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddq %xmm1, %xmm0
+; X64-SSE-NEXT: movq %xmm0, %rax
+; X64-SSE-NEXT: # kill: def $ax killed $ax killed $rax
; X64-SSE-NEXT: retq
;
; X86-AVX1-LABEL: test_v8i16_v8i8:
; X86-AVX1: # %bb.0:
; X86-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
; X86-AVX1-NEXT: retl
@@ -1566,19 +1517,21 @@ define i16 @test_v8i16_v8i8(<8 x i16> %a0) nounwind {
; X64-AVX1-LABEL: test_v8i16_v8i8:
; X64-AVX1: # %bb.0:
; X64-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX1-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $rax
; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: test_v8i16_v8i8:
; X86-AVX2: # %bb.0:
; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; X86-AVX2-NEXT: retl
@@ -1586,21 +1539,23 @@ define i16 @test_v8i16_v8i8(<8 x i16> %a0) nounwind {
; X64-AVX2-LABEL: test_v8i16_v8i8:
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; X64-AVX2-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vmovd %xmm0, %eax
-; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $rax
; X64-AVX2-NEXT: retq
;
; AVX512-LABEL: test_v8i16_v8i8:
; AVX512: # %bb.0:
; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $rax
; AVX512-NEXT: retq
%1 = and <8 x i16> %a0, <i16 0, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64>
%2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %1)
@@ -1610,11 +1565,10 @@ define i16 @test_v8i16_v8i8(<8 x i16> %a0) nounwind {
define i16 @test_v16i16_v16i8(<16 x i16> %a0) nounwind {
; X86-SSE2-LABEL: test_v16i16_v16i8:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [0,1,2,4,8,16,32,64]
; X86-SSE2-NEXT: pand %xmm2, %xmm1
; X86-SSE2-NEXT: pand %xmm2, %xmm0
-; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
-; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: paddw %xmm1, %xmm0
; X86-SSE2-NEXT: pxor %xmm1, %xmm1
; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
@@ -1625,26 +1579,24 @@ define i16 @test_v16i16_v16i8(<16 x i16> %a0) nounwind {
;
; X64-SSE2-LABEL: test_v16i16_v16i8:
; X64-SSE2: # %bb.0:
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [0,1,2,4,8,16,32,64]
; X64-SSE2-NEXT: pand %xmm2, %xmm1
; X64-SSE2-NEXT: pand %xmm2, %xmm0
-; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
-; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT: paddw %xmm1, %xmm0
; X64-SSE2-NEXT: pxor %xmm1, %xmm1
; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; X64-SSE2-NEXT: paddq %xmm1, %xmm0
-; X64-SSE2-NEXT: movd %xmm0, %eax
-; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $rax
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v16i16_v16i8:
; X86-SSE4: # %bb.0:
-; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; X86-SSE4-NEXT: pmovsxbw {{.*#+}} xmm2 = [0,1,2,4,8,16,32,64]
; X86-SSE4-NEXT: pand %xmm2, %xmm1
; X86-SSE4-NEXT: pand %xmm2, %xmm0
-; X86-SSE4-NEXT: packuswb %xmm1, %xmm0
-; X86-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE4-NEXT: paddw %xmm1, %xmm0
; X86-SSE4-NEXT: pxor %xmm1, %xmm1
; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
@@ -1655,25 +1607,23 @@ define i16 @test_v16i16_v16i8(<16 x i16> %a0) nounwind {
;
; X64-SSE4-LABEL: test_v16i16_v16i8:
; X64-SSE4: # %bb.0:
-; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; X64-SSE4-NEXT: pmovsxbw {{.*#+}} xmm2 = [0,1,2,4,8,16,32,64]
; X64-SSE4-NEXT: pand %xmm2, %xmm1
; X64-SSE4-NEXT: pand %xmm2, %xmm0
-; X64-SSE4-NEXT: packuswb %xmm1, %xmm0
-; X64-SSE4-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE4-NEXT: paddw %xmm1, %xmm0
; X64-SSE4-NEXT: pxor %xmm1, %xmm1
; X64-SSE4-NEXT: psadbw %xmm0, %xmm1
; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; X64-SSE4-NEXT: paddq %xmm1, %xmm0
-; X64-SSE4-NEXT: movd %xmm0, %eax
-; X64-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: # kill: def $ax killed $ax killed $rax
; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: test_v16i16_v16i8:
; X86-AVX1: # %bb.0:
; X86-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1687,25 +1637,23 @@ define i16 @test_v16i16_v16i8(<16 x i16> %a0) nounwind {
; X64-AVX1: # %bb.0:
; X64-AVX1-NEXT: vandps {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; X64-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $rax
; X64-AVX1-NEXT: vzeroupper
; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: test_v16i16_v16i8:
; X86-AVX2: # %bb.0:
; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %ymm0, %ymm0
-; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X86-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0
; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
@@ -1716,44 +1664,30 @@ define i16 @test_v16i16_v16i8(<16 x i16> %a0) nounwind {
; X64-AVX2-LABEL: test_v16i16_v16i8:
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
-; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X64-AVX2-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vmovd %xmm0, %eax
-; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $rax
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
-; AVX512BW-LABEL: test_v16i16_v16i8:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: # kill: def $ymm0 killed $ymm0 def $zmm0
-; AVX512BW-NEXT: vpmovwb %zmm0, %ymm0
-; AVX512BW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BW-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vmovd %xmm0, %eax
-; AVX512BW-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
-;
-; AVX512BWVL-LABEL: test_v16i16_v16i8:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpmovwb %ymm0, %xmm0
-; AVX512BWVL-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BWVL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BWVL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vmovd %xmm0, %eax
-; AVX512BWVL-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512BWVL-NEXT: vzeroupper
-; AVX512BWVL-NEXT: retq
+; AVX512-LABEL: test_v16i16_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%1 = and <16 x i16> %a0, <i16 0, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64, i16 0, i16 1, i16 2, i16 4, i16 8, i16 16, i16 32, i16 64>
%2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %1)
ret i16 %2
@@ -1767,19 +1701,21 @@ define i16 @test_v32i16_v32i8(<32 x i16> %a0) nounwind {
; X86-SSE-NEXT: andl $-16, %esp
; X86-SSE-NEXT: subl $16, %esp
; X86-SSE-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE-NEXT: psrlw $8, %xmm1
+; X86-SSE-NEXT: psrlw $8, %xmm2
; X86-SSE-NEXT: psrlw $8, %xmm0
-; X86-SSE-NEXT: packuswb %xmm1, %xmm0
+; X86-SSE-NEXT: paddw %xmm2, %xmm0
; X86-SSE-NEXT: psrlw $8, %xmm3
-; X86-SSE-NEXT: psrlw $8, %xmm2
-; X86-SSE-NEXT: packuswb %xmm3, %xmm2
-; X86-SSE-NEXT: pxor %xmm1, %xmm1
-; X86-SSE-NEXT: psadbw %xmm1, %xmm2
-; X86-SSE-NEXT: psadbw %xmm1, %xmm0
-; X86-SSE-NEXT: paddq %xmm2, %xmm0
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE-NEXT: paddq %xmm0, %xmm1
-; X86-SSE-NEXT: movd %xmm1, %eax
+; X86-SSE-NEXT: psrlw $8, %xmm1
+; X86-SSE-NEXT: paddw %xmm3, %xmm1
+; X86-SSE-NEXT: paddw %xmm0, %xmm1
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE-NEXT: paddw %xmm1, %xmm0
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE-NEXT: paddw %xmm0, %xmm1
+; X86-SSE-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE-NEXT: psrld $16, %xmm0
+; X86-SSE-NEXT: paddw %xmm1, %xmm0
+; X86-SSE-NEXT: movd %xmm0, %eax
; X86-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE-NEXT: movl %ebp, %esp
; X86-SSE-NEXT: popl %ebp
@@ -1787,55 +1723,78 @@ define i16 @test_v32i16_v32i8(<32 x i16> %a0) nounwind {
;
; X64-SSE-LABEL: test_v32i16_v32i8:
; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: psrlw $8, %xmm1
+; X64-SSE-NEXT: psrlw $8, %xmm2
; X64-SSE-NEXT: psrlw $8, %xmm0
-; X64-SSE-NEXT: packuswb %xmm1, %xmm0
+; X64-SSE-NEXT: paddw %xmm2, %xmm0
; X64-SSE-NEXT: psrlw $8, %xmm3
-; X64-SSE-NEXT: psrlw $8, %xmm2
-; X64-SSE-NEXT: packuswb %xmm3, %xmm2
-; X64-SSE-NEXT: pxor %xmm1, %xmm1
-; X64-SSE-NEXT: psadbw %xmm1, %xmm2
-; X64-SSE-NEXT: psadbw %xmm1, %xmm0
-; X64-SSE-NEXT: paddq %xmm2, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE-NEXT: paddq %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
+; X64-SSE-NEXT: psrlw $8, %xmm1
+; X64-SSE-NEXT: paddw %xmm3, %xmm1
+; X64-SSE-NEXT: paddw %xmm0, %xmm1
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddw %xmm1, %xmm0
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE-NEXT: paddw %xmm0, %xmm1
+; X64-SSE-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE-NEXT: psrld $16, %xmm0
+; X64-SSE-NEXT: paddw %xmm1, %xmm0
+; X64-SSE-NEXT: movd %xmm0, %eax
; X64-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SSE-NEXT: retq
;
-; AVX1-LABEL: test_v32i16_v32i8:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2
-; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpsrlw $8, %xmm2, %xmm2
-; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT: vpackuswb %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX1-NEXT: vzeroupper
-; AVX1-NEXT: ret{{[l|q]}}
+; AVX1-SLOW-LABEL: test_v32i16_v32i8:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vpsrlw $8, %xmm1, %xmm2
+; AVX1-SLOW-NEXT: vpsrlw $8, %xmm0, %xmm3
+; AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm3, %xmm2
+; AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-SLOW-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-SLOW-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-SLOW-NEXT: vzeroupper
+; AVX1-SLOW-NEXT: ret{{[l|q]}}
+;
+; AVX1-FAST-LABEL: test_v32i16_v32i8:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vpsrlw $8, %xmm1, %xmm2
+; AVX1-FAST-NEXT: vpsrlw $8, %xmm0, %xmm3
+; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm3, %xmm2
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
+; AVX1-FAST-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
+; AVX1-FAST-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-FAST-NEXT: vzeroupper
+; AVX1-FAST-NEXT: ret{{[l|q]}}
;
; AVX2-LABEL: test_v32i16_v32i8:
; AVX2: # %bb.0:
; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
-; AVX2-NEXT: vpackuswb %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: vzeroupper
@@ -1844,15 +1803,16 @@ define i16 @test_v32i16_v32i8(<32 x i16> %a0) nounwind {
; AVX512-LABEL: test_v32i16_v32i8:
; AVX512: # %bb.0:
; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512-NEXT: vpmovwb %zmm0, %ymm0
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $rax
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%1 = lshr <32 x i16> %a0, <i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8, i16 8>
@@ -1868,30 +1828,32 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm3 = [127,127,127,127,127,127,127,127]
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: packuswb %xmm1, %xmm0
; X86-SSE2-NEXT: pand %xmm3, %xmm2
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm4
-; X86-SSE2-NEXT: pand %xmm3, %xmm4
-; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm1
; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: packuswb %xmm4, %xmm1
-; X86-SSE2-NEXT: paddb %xmm0, %xmm1
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm0
-; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: packuswb %xmm0, %xmm2
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm0
+; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm4
+; X86-SSE2-NEXT: pand %xmm3, %xmm4
+; X86-SSE2-NEXT: paddw %xmm0, %xmm4
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm0
; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: pand 56(%ebp), %xmm3
-; X86-SSE2-NEXT: packuswb %xmm0, %xmm3
-; X86-SSE2-NEXT: paddb %xmm2, %xmm3
-; X86-SSE2-NEXT: pxor %xmm0, %xmm0
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm3
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE2-NEXT: paddq %xmm3, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X86-SSE2-NEXT: paddq %xmm1, %xmm0
+; X86-SSE2-NEXT: paddw %xmm2, %xmm0
+; X86-SSE2-NEXT: paddw %xmm4, %xmm0
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm2
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: paddw %xmm1, %xmm2
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE2-NEXT: pand %xmm3, %xmm1
+; X86-SSE2-NEXT: pand 8(%ebp), %xmm3
+; X86-SSE2-NEXT: paddw %xmm1, %xmm3
+; X86-SSE2-NEXT: paddw %xmm2, %xmm3
+; X86-SSE2-NEXT: paddw %xmm0, %xmm3
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X86-SSE2-NEXT: paddw %xmm3, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT: paddw %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: psrld $16, %xmm0
+; X86-SSE2-NEXT: paddw %xmm1, %xmm0
; X86-SSE2-NEXT: movd %xmm0, %eax
; X86-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE2-NEXT: movl %ebp, %esp
@@ -1901,26 +1863,28 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; X64-SSE2-LABEL: test_v64i16_v64i8:
; X64-SSE2: # %bb.0:
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm8 = [127,127,127,127,127,127,127,127]
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm0
-; X64-SSE2-NEXT: packuswb %xmm1, %xmm0
-; X64-SSE2-NEXT: pand %xmm8, %xmm5
; X64-SSE2-NEXT: pand %xmm8, %xmm4
-; X64-SSE2-NEXT: packuswb %xmm5, %xmm4
-; X64-SSE2-NEXT: paddb %xmm0, %xmm4
-; X64-SSE2-NEXT: pand %xmm8, %xmm3
+; X64-SSE2-NEXT: pand %xmm8, %xmm0
+; X64-SSE2-NEXT: paddw %xmm4, %xmm0
+; X64-SSE2-NEXT: pand %xmm8, %xmm6
; X64-SSE2-NEXT: pand %xmm8, %xmm2
-; X64-SSE2-NEXT: packuswb %xmm3, %xmm2
+; X64-SSE2-NEXT: paddw %xmm6, %xmm2
+; X64-SSE2-NEXT: paddw %xmm0, %xmm2
+; X64-SSE2-NEXT: pand %xmm8, %xmm5
+; X64-SSE2-NEXT: pand %xmm8, %xmm1
+; X64-SSE2-NEXT: paddw %xmm5, %xmm1
; X64-SSE2-NEXT: pand %xmm8, %xmm7
-; X64-SSE2-NEXT: pand %xmm8, %xmm6
-; X64-SSE2-NEXT: packuswb %xmm7, %xmm6
-; X64-SSE2-NEXT: paddb %xmm2, %xmm6
-; X64-SSE2-NEXT: pxor %xmm0, %xmm0
-; X64-SSE2-NEXT: psadbw %xmm0, %xmm6
-; X64-SSE2-NEXT: psadbw %xmm0, %xmm4
-; X64-SSE2-NEXT: paddq %xmm6, %xmm4
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
-; X64-SSE2-NEXT: paddq %xmm4, %xmm0
+; X64-SSE2-NEXT: pand %xmm8, %xmm3
+; X64-SSE2-NEXT: paddw %xmm7, %xmm3
+; X64-SSE2-NEXT: paddw %xmm1, %xmm3
+; X64-SSE2-NEXT: paddw %xmm2, %xmm3
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X64-SSE2-NEXT: paddw %xmm3, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE2-NEXT: paddw %xmm0, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT: psrld $16, %xmm0
+; X64-SSE2-NEXT: paddw %xmm1, %xmm0
; X64-SSE2-NEXT: movd %xmm0, %eax
; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SSE2-NEXT: retq
@@ -1932,30 +1896,32 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
; X86-SSE4-NEXT: pmovsxbw {{.*#+}} xmm3 = [127,127,127,127,127,127,127,127]
-; X86-SSE4-NEXT: pand %xmm3, %xmm1
; X86-SSE4-NEXT: pand %xmm3, %xmm0
-; X86-SSE4-NEXT: packuswb %xmm1, %xmm0
; X86-SSE4-NEXT: pand %xmm3, %xmm2
-; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm4
-; X86-SSE4-NEXT: pand %xmm3, %xmm4
-; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm1
; X86-SSE4-NEXT: pand %xmm3, %xmm1
-; X86-SSE4-NEXT: packuswb %xmm4, %xmm1
-; X86-SSE4-NEXT: paddb %xmm0, %xmm1
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm0
-; X86-SSE4-NEXT: pand %xmm3, %xmm0
-; X86-SSE4-NEXT: packuswb %xmm0, %xmm2
-; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm0
+; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm4
+; X86-SSE4-NEXT: pand %xmm3, %xmm4
+; X86-SSE4-NEXT: paddw %xmm0, %xmm4
+; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm0
; X86-SSE4-NEXT: pand %xmm3, %xmm0
-; X86-SSE4-NEXT: pand 56(%ebp), %xmm3
-; X86-SSE4-NEXT: packuswb %xmm0, %xmm3
-; X86-SSE4-NEXT: paddb %xmm2, %xmm3
-; X86-SSE4-NEXT: pxor %xmm0, %xmm0
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm3
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE4-NEXT: paddq %xmm3, %xmm1
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X86-SSE4-NEXT: paddq %xmm1, %xmm0
+; X86-SSE4-NEXT: paddw %xmm2, %xmm0
+; X86-SSE4-NEXT: paddw %xmm4, %xmm0
+; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm2
+; X86-SSE4-NEXT: pand %xmm3, %xmm2
+; X86-SSE4-NEXT: paddw %xmm1, %xmm2
+; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE4-NEXT: pand %xmm3, %xmm1
+; X86-SSE4-NEXT: pand 8(%ebp), %xmm3
+; X86-SSE4-NEXT: paddw %xmm1, %xmm3
+; X86-SSE4-NEXT: paddw %xmm2, %xmm3
+; X86-SSE4-NEXT: paddw %xmm0, %xmm3
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X86-SSE4-NEXT: paddw %xmm3, %xmm0
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: paddw %xmm0, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE4-NEXT: psrld $16, %xmm0
+; X86-SSE4-NEXT: paddw %xmm1, %xmm0
; X86-SSE4-NEXT: movd %xmm0, %eax
; X86-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE4-NEXT: movl %ebp, %esp
@@ -1965,91 +1931,153 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; X64-SSE4-LABEL: test_v64i16_v64i8:
; X64-SSE4: # %bb.0:
; X64-SSE4-NEXT: pmovsxbw {{.*#+}} xmm8 = [127,127,127,127,127,127,127,127]
-; X64-SSE4-NEXT: pand %xmm8, %xmm1
-; X64-SSE4-NEXT: pand %xmm8, %xmm0
-; X64-SSE4-NEXT: packuswb %xmm1, %xmm0
-; X64-SSE4-NEXT: pand %xmm8, %xmm5
; X64-SSE4-NEXT: pand %xmm8, %xmm4
-; X64-SSE4-NEXT: packuswb %xmm5, %xmm4
-; X64-SSE4-NEXT: paddb %xmm0, %xmm4
-; X64-SSE4-NEXT: pand %xmm8, %xmm3
+; X64-SSE4-NEXT: pand %xmm8, %xmm0
+; X64-SSE4-NEXT: paddw %xmm4, %xmm0
+; X64-SSE4-NEXT: pand %xmm8, %xmm6
; X64-SSE4-NEXT: pand %xmm8, %xmm2
-; X64-SSE4-NEXT: packuswb %xmm3, %xmm2
+; X64-SSE4-NEXT: paddw %xmm6, %xmm2
+; X64-SSE4-NEXT: paddw %xmm0, %xmm2
+; X64-SSE4-NEXT: pand %xmm8, %xmm5
+; X64-SSE4-NEXT: pand %xmm8, %xmm1
+; X64-SSE4-NEXT: paddw %xmm5, %xmm1
; X64-SSE4-NEXT: pand %xmm8, %xmm7
-; X64-SSE4-NEXT: pand %xmm8, %xmm6
-; X64-SSE4-NEXT: packuswb %xmm7, %xmm6
-; X64-SSE4-NEXT: paddb %xmm2, %xmm6
-; X64-SSE4-NEXT: pxor %xmm0, %xmm0
-; X64-SSE4-NEXT: psadbw %xmm0, %xmm6
-; X64-SSE4-NEXT: psadbw %xmm0, %xmm4
-; X64-SSE4-NEXT: paddq %xmm6, %xmm4
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm4, %xmm0
+; X64-SSE4-NEXT: pand %xmm8, %xmm3
+; X64-SSE4-NEXT: paddw %xmm7, %xmm3
+; X64-SSE4-NEXT: paddw %xmm1, %xmm3
+; X64-SSE4-NEXT: paddw %xmm2, %xmm3
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X64-SSE4-NEXT: paddw %xmm3, %xmm0
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: paddw %xmm0, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE4-NEXT: psrld $16, %xmm0
+; X64-SSE4-NEXT: paddw %xmm1, %xmm0
; X64-SSE4-NEXT: movd %xmm0, %eax
; X64-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SSE4-NEXT: retq
;
-; X86-AVX1-LABEL: test_v64i16_v64i8:
-; X86-AVX1: # %bb.0:
-; X86-AVX1-NEXT: pushl %ebp
-; X86-AVX1-NEXT: movl %esp, %ebp
-; X86-AVX1-NEXT: andl $-32, %esp
-; X86-AVX1-NEXT: subl $32, %esp
-; X86-AVX1-NEXT: vbroadcastss {{.*#+}} ymm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-AVX1-NEXT: vandps %ymm3, %ymm1, %ymm1
-; X86-AVX1-NEXT: vandps %ymm3, %ymm0, %ymm0
-; X86-AVX1-NEXT: vandps %ymm3, %ymm2, %ymm2
-; X86-AVX1-NEXT: vandps 8(%ebp), %ymm3, %ymm3
-; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4
-; X86-AVX1-NEXT: vpackuswb %xmm4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
-; X86-AVX1-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
-; X86-AVX1-NEXT: vpackuswb %xmm4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
-; X86-AVX1-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vmovd %xmm0, %eax
-; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-AVX1-NEXT: movl %ebp, %esp
-; X86-AVX1-NEXT: popl %ebp
-; X86-AVX1-NEXT: vzeroupper
-; X86-AVX1-NEXT: retl
+; X86-AVX1-SLOW-LABEL: test_v64i16_v64i8:
+; X86-AVX1-SLOW: # %bb.0:
+; X86-AVX1-SLOW-NEXT: pushl %ebp
+; X86-AVX1-SLOW-NEXT: movl %esp, %ebp
+; X86-AVX1-SLOW-NEXT: andl $-32, %esp
+; X86-AVX1-SLOW-NEXT: subl $32, %esp
+; X86-AVX1-SLOW-NEXT: vbroadcastss {{.*#+}} ymm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; X86-AVX1-SLOW-NEXT: vandps %ymm3, %ymm1, %ymm1
+; X86-AVX1-SLOW-NEXT: vandps %ymm3, %ymm0, %ymm0
+; X86-AVX1-SLOW-NEXT: vandps %ymm3, %ymm2, %ymm2
+; X86-AVX1-SLOW-NEXT: vandps 8(%ebp), %ymm3, %ymm3
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm4
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm5
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm6
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm6, %xmm5, %xmm5
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm5, %xmm4, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm1, %xmm1
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-SLOW-NEXT: movl %ebp, %esp
+; X86-AVX1-SLOW-NEXT: popl %ebp
+; X86-AVX1-SLOW-NEXT: vzeroupper
+; X86-AVX1-SLOW-NEXT: retl
;
-; X64-AVX1-LABEL: test_v64i16_v64i8:
-; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vbroadcastss {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-AVX1-NEXT: vandps %ymm4, %ymm0, %ymm0
-; X64-AVX1-NEXT: vandps %ymm4, %ymm2, %ymm2
-; X64-AVX1-NEXT: vandps %ymm4, %ymm1, %ymm1
-; X64-AVX1-NEXT: vandps %ymm4, %ymm3, %ymm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
-; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm3, %xmm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
-; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4
-; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm2, %xmm2
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
-; X64-AVX1-NEXT: vpackuswb %xmm4, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vmovd %xmm0, %eax
-; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-AVX1-NEXT: vzeroupper
-; X64-AVX1-NEXT: retq
+; X64-AVX1-SLOW-LABEL: test_v64i16_v64i8:
+; X64-AVX1-SLOW: # %bb.0:
+; X64-AVX1-SLOW-NEXT: vbroadcastss {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm0, %ymm0
+; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm2, %ymm2
+; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm1, %ymm1
+; X64-AVX1-SLOW-NEXT: vandps %ymm4, %ymm3, %ymm3
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm5, %xmm6, %xmm5
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm1, %xmm1
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm0, %xmm0
+; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-SLOW-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX1-SLOW-NEXT: vzeroupper
+; X64-AVX1-SLOW-NEXT: retq
+;
+; X86-AVX1-FAST-LABEL: test_v64i16_v64i8:
+; X86-AVX1-FAST: # %bb.0:
+; X86-AVX1-FAST-NEXT: pushl %ebp
+; X86-AVX1-FAST-NEXT: movl %esp, %ebp
+; X86-AVX1-FAST-NEXT: andl $-32, %esp
+; X86-AVX1-FAST-NEXT: subl $32, %esp
+; X86-AVX1-FAST-NEXT: vbroadcastss {{.*#+}} ymm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; X86-AVX1-FAST-NEXT: vandps %ymm3, %ymm1, %ymm1
+; X86-AVX1-FAST-NEXT: vandps %ymm3, %ymm0, %ymm0
+; X86-AVX1-FAST-NEXT: vandps %ymm3, %ymm2, %ymm2
+; X86-AVX1-FAST-NEXT: vandps 8(%ebp), %ymm3, %ymm3
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm4
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm5
+; X86-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm6
+; X86-AVX1-FAST-NEXT: vpaddw %xmm6, %xmm5, %xmm5
+; X86-AVX1-FAST-NEXT: vpaddw %xmm5, %xmm4, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vpaddw %xmm3, %xmm1, %xmm1
+; X86-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-FAST-NEXT: movl %ebp, %esp
+; X86-AVX1-FAST-NEXT: popl %ebp
+; X86-AVX1-FAST-NEXT: vzeroupper
+; X86-AVX1-FAST-NEXT: retl
+;
+; X64-AVX1-FAST-LABEL: test_v64i16_v64i8:
+; X64-AVX1-FAST: # %bb.0:
+; X64-AVX1-FAST-NEXT: vbroadcastss {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm0, %ymm0
+; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm2, %ymm2
+; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm1, %ymm1
+; X64-AVX1-FAST-NEXT: vandps %ymm4, %ymm3, %ymm3
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-FAST-NEXT: vpaddw %xmm5, %xmm6, %xmm5
+; X64-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X64-AVX1-FAST-NEXT: vpaddw %xmm3, %xmm1, %xmm1
+; X64-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-AVX1-FAST-NEXT: vzeroupper
+; X64-AVX1-FAST-NEXT: retq
;
; X86-AVX2-LABEL: test_v64i16_v64i8:
; X86-AVX2: # %bb.0:
@@ -2058,20 +2086,21 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; X86-AVX2-NEXT: andl $-32, %esp
; X86-AVX2-NEXT: subl $32, %esp
; X86-AVX2-NEXT: vpbroadcastw {{.*#+}} ymm3 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
; X86-AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpackuswb %ymm1, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpand 8(%ebp), %ymm3, %ymm1
-; X86-AVX2-NEXT: vpackuswb %ymm1, %ymm2, %ymm1
-; X86-AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpaddw %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpand 8(%ebp), %ymm3, %ymm2
+; X86-AVX2-NEXT: vpaddw %ymm2, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
+; X86-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; X86-AVX2-NEXT: movl %ebp, %esp
@@ -2082,20 +2111,21 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; X64-AVX2-LABEL: test_v64i16_v64i8:
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: vpbroadcastw {{.*#+}} ymm4 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
-; X64-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
-; X64-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpackuswb %ymm1, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm1
; X64-AVX2-NEXT: vpand %ymm4, %ymm2, %ymm2
-; X64-AVX2-NEXT: vpackuswb %ymm1, %ymm2, %ymm1
-; X64-AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0
-; X64-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
-; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpaddw %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpand %ymm4, %ymm3, %ymm2
+; X64-AVX2-NEXT: vpand %ymm4, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpaddw %ymm2, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
+; X64-AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovd %xmm0, %eax
; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; X64-AVX2-NEXT: vzeroupper
@@ -2103,10 +2133,10 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
;
; AVX512-LABEL: test_v64i16_v64i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpmovwb %zmm0, %ymm0
-; AVX512-NEXT: vpmovwb %zmm1, %ymm1
-; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0
-; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm0, %zmm0
+; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm1
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
@@ -2115,8 +2145,8 @@ define i16 @test_v64i16_v64i8(<64 x i16> %a0) nounwind {
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $rax
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%1 = and <64 x i16> %a0, <i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127, i16 127>
@@ -2151,4 +2181,6 @@ declare i8 @llvm.vector.reduce.add.v64i8(<64 x i8>)
declare i8 @llvm.vector.reduce.add.v128i8(<128 x i8>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; AVX: {{.*}}
+; AVX1: {{.*}}
+; SSE: {{.*}}
; SSE2: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll b/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll
index 3b041304e252d..813114552505c 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-sext.ll
@@ -379,7 +379,7 @@ define i64 @test_v8i64_v8i8(<8 x i8> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovsxbq %xmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -403,7 +403,7 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
; X86-SSE2-NEXT: pcmpgtd %xmm2, %xmm1
; X86-SSE2-NEXT: movdqu %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-SSE2-NEXT: movdqa %xmm2, %xmm4
-; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm0[0],xmm7[1],xmm0[1],xmm7[2],xmm0[2],xmm7[3],xmm0[3]
; X86-SSE2-NEXT: psrad $24, %xmm7
@@ -411,7 +411,7 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
; X86-SSE2-NEXT: pcmpgtd %xmm7, %xmm5
; X86-SSE2-NEXT: movdqu %xmm5, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-SSE2-NEXT: movdqa %xmm7, %xmm1
-; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm5[2],xmm1[3],xmm5[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
; X86-SSE2-NEXT: paddq %xmm4, %xmm1
; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
; X86-SSE2-NEXT: psrad $24, %xmm3
@@ -419,23 +419,23 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
; X86-SSE2-NEXT: pcmpgtd %xmm3, %xmm4
; X86-SSE2-NEXT: movdqu %xmm4, (%esp) # 16-byte Spill
; X86-SSE2-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm4[2],xmm6[3],xmm4[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
; X86-SSE2-NEXT: psrad $24, %xmm4
; X86-SSE2-NEXT: pxor %xmm5, %xmm5
; X86-SSE2-NEXT: pcmpgtd %xmm4, %xmm5
; X86-SSE2-NEXT: movdqa %xmm4, %xmm0
-; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm5[2],xmm0[3],xmm5[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
; X86-SSE2-NEXT: paddq %xmm6, %xmm0
; X86-SSE2-NEXT: paddq %xmm1, %xmm0
; X86-SSE2-NEXT: movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
; X86-SSE2-NEXT: movdqu {{[-0-9]+}}(%e{{[sb]}}p), %xmm1 # 16-byte Reload
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm1[0],xmm7[1],xmm1[1]
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm7 = xmm7[2],xmm1[2],xmm7[3],xmm1[3]
; X86-SSE2-NEXT: paddq %xmm2, %xmm7
; X86-SSE2-NEXT: movdqu (%esp), %xmm1 # 16-byte Reload
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm1[2],xmm3[3],xmm1[3]
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
; X86-SSE2-NEXT: paddq %xmm3, %xmm4
; X86-SSE2-NEXT: paddq %xmm7, %xmm4
; X86-SSE2-NEXT: paddq %xmm0, %xmm4
@@ -456,33 +456,33 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
; X64-SSE2-NEXT: pxor %xmm3, %xmm3
; X64-SSE2-NEXT: pcmpgtd %xmm2, %xmm3
; X64-SSE2-NEXT: movdqa %xmm2, %xmm5
-; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm3[2],xmm5[3],xmm3[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1]
; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3],xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1],xmm0[2],xmm6[2],xmm0[3],xmm6[3]
; X64-SSE2-NEXT: psrad $24, %xmm0
; X64-SSE2-NEXT: pxor %xmm7, %xmm7
; X64-SSE2-NEXT: pcmpgtd %xmm0, %xmm7
; X64-SSE2-NEXT: movdqa %xmm0, %xmm8
-; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm8 = xmm8[2],xmm7[2],xmm8[3],xmm7[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm8 = xmm8[0],xmm7[0],xmm8[1],xmm7[1]
; X64-SSE2-NEXT: paddq %xmm5, %xmm8
; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
; X64-SSE2-NEXT: psrad $24, %xmm4
; X64-SSE2-NEXT: pxor %xmm5, %xmm5
; X64-SSE2-NEXT: pcmpgtd %xmm4, %xmm5
; X64-SSE2-NEXT: movdqa %xmm4, %xmm9
-; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm9 = xmm9[2],xmm5[2],xmm9[3],xmm5[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm5[0],xmm9[1],xmm5[1]
; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4,4,5,5,6,6,7,7]
; X64-SSE2-NEXT: psrad $24, %xmm6
; X64-SSE2-NEXT: pcmpgtd %xmm6, %xmm1
; X64-SSE2-NEXT: movdqa %xmm6, %xmm10
-; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm1[2],xmm10[3],xmm1[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm10 = xmm10[0],xmm1[0],xmm10[1],xmm1[1]
; X64-SSE2-NEXT: paddq %xmm9, %xmm10
; X64-SSE2-NEXT: paddq %xmm8, %xmm10
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm7[0],xmm0[1],xmm7[1]
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm7[2],xmm0[3],xmm7[3]
; X64-SSE2-NEXT: paddq %xmm2, %xmm0
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm5[0],xmm4[1],xmm5[1]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1]
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm5[2],xmm4[3],xmm5[3]
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm6 = xmm6[2],xmm1[2],xmm6[3],xmm1[3]
; X64-SSE2-NEXT: paddq %xmm4, %xmm6
; X64-SSE2-NEXT: paddq %xmm0, %xmm6
; X64-SSE2-NEXT: paddq %xmm10, %xmm6
@@ -493,95 +493,95 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
;
; X86-SSE4-LABEL: test_v16i64_v16i8:
; X86-SSE4: # %bb.0:
-; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE4-NEXT: pmovsxbq %xmm0, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: pmovsxbq %xmm2, %xmm2
+; X86-SSE4-NEXT: paddq %xmm1, %xmm2
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; X86-SSE4-NEXT: pmovsxbq %xmm1, %xmm3
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: pmovsxbq %xmm1, %xmm1
+; X86-SSE4-NEXT: paddq %xmm3, %xmm1
+; X86-SSE4-NEXT: paddq %xmm2, %xmm1
; X86-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE4-NEXT: psrldq {{.*#+}} xmm2 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: pmovsxbq %xmm2, %xmm2
; X86-SSE4-NEXT: movdqa %xmm0, %xmm3
-; X86-SSE4-NEXT: pmovsxbq %xmm0, %xmm4
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm0[3,3,3,3]
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,1,1]
-; X86-SSE4-NEXT: psrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X86-SSE4-NEXT: pmovsxbq %xmm0, %xmm0
-; X86-SSE4-NEXT: psrld $16, %xmm1
-; X86-SSE4-NEXT: pmovsxbq %xmm1, %xmm1
-; X86-SSE4-NEXT: paddq %xmm0, %xmm1
-; X86-SSE4-NEXT: psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X86-SSE4-NEXT: pmovsxbq %xmm2, %xmm0
-; X86-SSE4-NEXT: psrlq $48, %xmm3
-; X86-SSE4-NEXT: pmovsxbq %xmm3, %xmm2
-; X86-SSE4-NEXT: paddq %xmm0, %xmm2
-; X86-SSE4-NEXT: paddq %xmm1, %xmm2
-; X86-SSE4-NEXT: pmovsxbq %xmm5, %xmm0
-; X86-SSE4-NEXT: paddq %xmm4, %xmm0
-; X86-SSE4-NEXT: pmovsxbq %xmm6, %xmm1
-; X86-SSE4-NEXT: pmovsxbq %xmm7, %xmm3
-; X86-SSE4-NEXT: paddq %xmm1, %xmm3
-; X86-SSE4-NEXT: paddq %xmm0, %xmm3
+; X86-SSE4-NEXT: psrld $16, %xmm3
+; X86-SSE4-NEXT: pmovsxbq %xmm3, %xmm3
; X86-SSE4-NEXT: paddq %xmm2, %xmm3
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE4-NEXT: psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: pmovsxbq %xmm2, %xmm2
+; X86-SSE4-NEXT: psrlq $48, %xmm0
+; X86-SSE4-NEXT: pmovsxbq %xmm0, %xmm0
+; X86-SSE4-NEXT: paddq %xmm2, %xmm0
; X86-SSE4-NEXT: paddq %xmm3, %xmm0
-; X86-SSE4-NEXT: movd %xmm0, %eax
-; X86-SSE4-NEXT: pextrd $1, %xmm0, %edx
+; X86-SSE4-NEXT: paddq %xmm1, %xmm0
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm0, %xmm1
+; X86-SSE4-NEXT: movd %xmm1, %eax
+; X86-SSE4-NEXT: pextrd $1, %xmm1, %edx
; X86-SSE4-NEXT: retl
;
; X64-SSE4-LABEL: test_v16i64_v16i8:
; X64-SSE4: # %bb.0:
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE4-NEXT: pmovsxbq %xmm0, %xmm4
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm0[3,3,3,3]
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm7 = xmm0[1,1,1,1]
-; X64-SSE4-NEXT: psrldq {{.*#+}} xmm0 = xmm0[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-SSE4-NEXT: pmovsxbq %xmm0, %xmm0
-; X64-SSE4-NEXT: psrld $16, %xmm1
-; X64-SSE4-NEXT: pmovsxbq %xmm1, %xmm1
-; X64-SSE4-NEXT: paddq %xmm0, %xmm1
-; X64-SSE4-NEXT: psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-SSE4-NEXT: pmovsxbq %xmm2, %xmm0
-; X64-SSE4-NEXT: psrlq $48, %xmm3
-; X64-SSE4-NEXT: pmovsxbq %xmm3, %xmm2
-; X64-SSE4-NEXT: paddq %xmm0, %xmm2
+; X64-SSE4-NEXT: pmovsxbq %xmm0, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: pmovsxbq %xmm2, %xmm2
; X64-SSE4-NEXT: paddq %xmm1, %xmm2
-; X64-SSE4-NEXT: pmovsxbq %xmm5, %xmm0
-; X64-SSE4-NEXT: paddq %xmm4, %xmm0
-; X64-SSE4-NEXT: pmovsxbq %xmm6, %xmm1
-; X64-SSE4-NEXT: pmovsxbq %xmm7, %xmm3
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; X64-SSE4-NEXT: pmovsxbq %xmm1, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: pmovsxbq %xmm3, %xmm3
; X64-SSE4-NEXT: paddq %xmm1, %xmm3
-; X64-SSE4-NEXT: paddq %xmm0, %xmm3
; X64-SSE4-NEXT: paddq %xmm2, %xmm3
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE4-NEXT: psrldq {{.*#+}} xmm1 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: pmovsxbq %xmm1, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X64-SSE4-NEXT: psrld $16, %xmm2
+; X64-SSE4-NEXT: pmovsxbq %xmm2, %xmm2
+; X64-SSE4-NEXT: paddq %xmm1, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE4-NEXT: psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: pmovsxbq %xmm1, %xmm1
+; X64-SSE4-NEXT: psrlq $48, %xmm0
+; X64-SSE4-NEXT: pmovsxbq %xmm0, %xmm0
+; X64-SSE4-NEXT: paddq %xmm1, %xmm0
+; X64-SSE4-NEXT: paddq %xmm2, %xmm0
; X64-SSE4-NEXT: paddq %xmm3, %xmm0
-; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm0, %xmm1
+; X64-SSE4-NEXT: movq %xmm1, %rax
; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: test_v16i64_v16i8:
; X86-AVX1: # %bb.0:
-; X86-AVX1-NEXT: vpsrld $16, %xmm0, %xmm1
-; X86-AVX1-NEXT: vpmovsxbq %xmm1, %xmm2
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-NEXT: vpmovsxbw %xmm1, %xmm3
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
-; X86-AVX1-NEXT: vpmovsxwq %xmm4, %xmm4
-; X86-AVX1-NEXT: vpaddq %xmm4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpmovsxbw %xmm0, %xmm4
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
+; X86-AVX1-NEXT: vpmovsxbw %xmm0, %xmm1
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X86-AVX1-NEXT: vpmovsxwq %xmm2, %xmm2
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpmovsxbw %xmm3, %xmm4
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
; X86-AVX1-NEXT: vpmovsxwq %xmm5, %xmm5
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[3,3,3,3]
-; X86-AVX1-NEXT: vpmovsxwq %xmm6, %xmm6
-; X86-AVX1-NEXT: vpaddq %xmm6, %xmm5, %xmm5
; X86-AVX1-NEXT: vpaddq %xmm5, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
-; X86-AVX1-NEXT: vpmovsxwq %xmm4, %xmm4
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
-; X86-AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm4, %xmm3
+; X86-AVX1-NEXT: vpmovsxbq %xmm0, %xmm5
+; X86-AVX1-NEXT: vpmovsxbq %xmm3, %xmm3
+; X86-AVX1-NEXT: vpaddq %xmm3, %xmm5, %xmm3
+; X86-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
+; X86-AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
; X86-AVX1-NEXT: vpmovsxbq %xmm0, %xmm0
-; X86-AVX1-NEXT: vpmovsxbq %xmm1, %xmm1
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
+; X86-AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
; X86-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; X86-AVX1-NEXT: vpmovsxwq %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[3,3,3,3]
+; X86-AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
+; X86-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
@@ -590,30 +590,30 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
;
; X64-AVX1-LABEL: test_v16i64_v16i8:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vpsrld $16, %xmm0, %xmm1
-; X64-AVX1-NEXT: vpmovsxbq %xmm1, %xmm1
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpmovsxbw %xmm2, %xmm3
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[1,1,1,1]
-; X64-AVX1-NEXT: vpmovsxwq %xmm4, %xmm4
-; X64-AVX1-NEXT: vpaddq %xmm4, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpmovsxbw %xmm0, %xmm4
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[3,3,3,3]
+; X64-AVX1-NEXT: vpmovsxbw %xmm0, %xmm1
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X64-AVX1-NEXT: vpmovsxwq %xmm2, %xmm2
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpmovsxbw %xmm3, %xmm4
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm4[2,3,2,3]
; X64-AVX1-NEXT: vpmovsxwq %xmm5, %xmm5
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm3[3,3,3,3]
-; X64-AVX1-NEXT: vpmovsxwq %xmm6, %xmm6
-; X64-AVX1-NEXT: vpaddq %xmm6, %xmm5, %xmm5
-; X64-AVX1-NEXT: vpaddq %xmm5, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[2,3,2,3]
-; X64-AVX1-NEXT: vpmovsxwq %xmm4, %xmm4
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[2,3,2,3]
-; X64-AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
-; X64-AVX1-NEXT: vpaddq %xmm3, %xmm4, %xmm3
+; X64-AVX1-NEXT: vpaddq %xmm5, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpmovsxbq %xmm0, %xmm5
+; X64-AVX1-NEXT: vpmovsxbq %xmm3, %xmm3
+; X64-AVX1-NEXT: vpaddq %xmm3, %xmm5, %xmm3
+; X64-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
+; X64-AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
; X64-AVX1-NEXT: vpmovsxbq %xmm0, %xmm0
-; X64-AVX1-NEXT: vpmovsxbq %xmm2, %xmm2
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[1,1,1,1]
+; X64-AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
; X64-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; X64-AVX1-NEXT: vpmovsxwq %xmm1, %xmm1
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[3,3,3,3]
+; X64-AVX1-NEXT: vpmovsxwq %xmm3, %xmm3
+; X64-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
@@ -670,7 +670,7 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
; AVX512-NEXT: vpmovsxwq %xmm0, %zmm0
; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -852,7 +852,7 @@ define i32 @test_v8i32_v8i8(<8 x i8> %a0) nounwind {
; AVX1-FAST-NEXT: vpmovsxbd %xmm0, %xmm1
; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
; AVX1-FAST-NEXT: vpmovsxbd %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm1, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
@@ -961,8 +961,7 @@ define i32 @test_v16i32_v16i8(<16 x i8> %a0) nounwind {
; AVX1-FAST-NEXT: vpmovsxbd %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm1, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: ret{{[l|q]}}
@@ -987,7 +986,7 @@ define i32 @test_v16i32_v16i8(<16 x i8> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovsxbd %xmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1006,28 +1005,28 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
; X86-SSE2-LABEL: test_v32i32_v32i8:
; X86-SSE2: # %bb.0:
; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
; X86-SSE2-NEXT: psrad $24, %xmm4
; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
-; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm3[4],xmm5[5],xmm3[5],xmm5[6],xmm3[6],xmm5[7],xmm3[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm3[0],xmm5[1],xmm3[1],xmm5[2],xmm3[2],xmm5[3],xmm3[3]
; X86-SSE2-NEXT: psrad $24, %xmm5
; X86-SSE2-NEXT: paddd %xmm4, %xmm5
; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm1[4],xmm6[5],xmm1[5],xmm6[6],xmm1[6],xmm6[7],xmm1[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3]
; X86-SSE2-NEXT: psrad $24, %xmm6
; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm0[8],xmm4[9],xmm0[9],xmm4[10],xmm0[10],xmm4[11],xmm0[11],xmm4[12],xmm0[12],xmm4[13],xmm0[13],xmm4[14],xmm0[14],xmm4[15],xmm0[15]
-; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm4[4],xmm0[5],xmm4[5],xmm0[6],xmm4[6],xmm0[7],xmm4[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1],xmm0[2],xmm4[2],xmm0[3],xmm4[3]
; X86-SSE2-NEXT: psrad $24, %xmm0
; X86-SSE2-NEXT: paddd %xmm6, %xmm0
; X86-SSE2-NEXT: paddd %xmm5, %xmm0
-; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
; X86-SSE2-NEXT: psrad $24, %xmm2
-; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0,0,1,1,2,2,3,3]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4,4,5,5,6,6,7,7]
; X86-SSE2-NEXT: psrad $24, %xmm3
; X86-SSE2-NEXT: paddd %xmm2, %xmm3
-; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
; X86-SSE2-NEXT: psrad $24, %xmm1
-; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
; X86-SSE2-NEXT: psrad $24, %xmm2
; X86-SSE2-NEXT: paddd %xmm1, %xmm2
; X86-SSE2-NEXT: paddd %xmm3, %xmm2
@@ -1042,28 +1041,28 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
; X64-SSE2-LABEL: test_v32i32_v32i8:
; X64-SSE2: # %bb.0:
; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
; X64-SSE2-NEXT: psrad $24, %xmm3
; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
-; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
; X64-SSE2-NEXT: psrad $24, %xmm5
; X64-SSE2-NEXT: paddd %xmm3, %xmm5
; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; X64-SSE2-NEXT: psrad $24, %xmm3
; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
-; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm0[4],xmm6[5],xmm0[5],xmm6[6],xmm0[6],xmm6[7],xmm0[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm0[0],xmm6[1],xmm0[1],xmm6[2],xmm0[2],xmm6[3],xmm0[3]
; X64-SSE2-NEXT: psrad $24, %xmm6
; X64-SSE2-NEXT: paddd %xmm3, %xmm6
; X64-SSE2-NEXT: paddd %xmm5, %xmm6
-; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4,4,5,5,6,6,7,7]
; X64-SSE2-NEXT: psrad $24, %xmm2
-; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm4[4],xmm3[5],xmm4[5],xmm3[6],xmm4[6],xmm3[7],xmm4[7]
; X64-SSE2-NEXT: psrad $24, %xmm3
; X64-SSE2-NEXT: paddd %xmm2, %xmm3
-; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4,4,5,5,6,6,7,7]
; X64-SSE2-NEXT: psrad $24, %xmm1
-; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4,4,5,5,6,6,7,7]
; X64-SSE2-NEXT: psrad $24, %xmm0
; X64-SSE2-NEXT: paddd %xmm1, %xmm0
; X64-SSE2-NEXT: paddd %xmm3, %xmm0
@@ -1077,23 +1076,23 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
;
; X86-SSE4-LABEL: test_v32i32_v32i8:
; X86-SSE4: # %bb.0:
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; X86-SSE4-NEXT: pmovsxbd %xmm2, %xmm2
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; X86-SSE4-NEXT: pmovsxbd %xmm3, %xmm3
; X86-SSE4-NEXT: paddd %xmm2, %xmm3
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X86-SSE4-NEXT: pmovsxbd %xmm2, %xmm4
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; X86-SSE4-NEXT: pmovsxbd %xmm2, %xmm2
+; X86-SSE4-NEXT: pmovsxbd %xmm1, %xmm4
+; X86-SSE4-NEXT: pmovsxbd %xmm0, %xmm2
; X86-SSE4-NEXT: paddd %xmm4, %xmm2
; X86-SSE4-NEXT: paddd %xmm3, %xmm2
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
; X86-SSE4-NEXT: pmovsxbd %xmm3, %xmm3
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
; X86-SSE4-NEXT: pmovsxbd %xmm4, %xmm4
; X86-SSE4-NEXT: paddd %xmm3, %xmm4
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
; X86-SSE4-NEXT: pmovsxbd %xmm1, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; X86-SSE4-NEXT: pmovsxbd %xmm0, %xmm0
; X86-SSE4-NEXT: paddd %xmm1, %xmm0
; X86-SSE4-NEXT: paddd %xmm4, %xmm0
@@ -1107,23 +1106,23 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
;
; X64-SSE4-LABEL: test_v32i32_v32i8:
; X64-SSE4: # %bb.0:
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; X64-SSE4-NEXT: pmovsxbd %xmm2, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; X64-SSE4-NEXT: pmovsxbd %xmm3, %xmm3
; X64-SSE4-NEXT: paddd %xmm2, %xmm3
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[3,3,3,3]
-; X64-SSE4-NEXT: pmovsxbd %xmm2, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
-; X64-SSE4-NEXT: pmovsxbd %xmm4, %xmm4
+; X64-SSE4-NEXT: pmovsxbd %xmm1, %xmm2
+; X64-SSE4-NEXT: pmovsxbd %xmm0, %xmm4
; X64-SSE4-NEXT: paddd %xmm2, %xmm4
; X64-SSE4-NEXT: paddd %xmm3, %xmm4
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
; X64-SSE4-NEXT: pmovsxbd %xmm2, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
; X64-SSE4-NEXT: pmovsxbd %xmm3, %xmm3
; X64-SSE4-NEXT: paddd %xmm2, %xmm3
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
; X64-SSE4-NEXT: pmovsxbd %xmm1, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; X64-SSE4-NEXT: pmovsxbd %xmm0, %xmm0
; X64-SSE4-NEXT: paddd %xmm1, %xmm0
; X64-SSE4-NEXT: paddd %xmm3, %xmm0
@@ -1138,27 +1137,27 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
; AVX1-SLOW-LABEL: test_v32i32_v32i8:
; AVX1-SLOW: # %bb.0:
; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; AVX1-SLOW-NEXT: vpmovsxbd %xmm2, %xmm2
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; AVX1-SLOW-NEXT: vpmovsxbd %xmm3, %xmm3
; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[3,3,3,3]
-; AVX1-SLOW-NEXT: vpmovsxbd %xmm3, %xmm3
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
-; AVX1-SLOW-NEXT: vpmovsxbd %xmm4, %xmm4
+; AVX1-SLOW-NEXT: vpmovsxbd %xmm1, %xmm3
+; AVX1-SLOW-NEXT: vpmovsxbd %xmm0, %xmm4
; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm4, %xmm3
-; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm2, %xmm2
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
; AVX1-SLOW-NEXT: vpmovsxbd %xmm3, %xmm3
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
; AVX1-SLOW-NEXT: vpmovsxbd %xmm4, %xmm4
; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
; AVX1-SLOW-NEXT: vpmovsxbd %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX1-SLOW-NEXT: vpmovsxbd %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm2, %xmm0
; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1170,29 +1169,28 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
; AVX1-FAST-LABEL: test_v32i32_v32i8:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; AVX1-FAST-NEXT: vpmovsxbd %xmm2, %xmm2
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
; AVX1-FAST-NEXT: vpmovsxbd %xmm3, %xmm3
; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[3,3,3,3]
-; AVX1-FAST-NEXT: vpmovsxbd %xmm3, %xmm3
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[3,3,3,3]
-; AVX1-FAST-NEXT: vpmovsxbd %xmm4, %xmm4
+; AVX1-FAST-NEXT: vpmovsxbd %xmm1, %xmm3
+; AVX1-FAST-NEXT: vpmovsxbd %xmm0, %xmm4
; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm4, %xmm3
-; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm2, %xmm2
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[2,3,2,3]
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
; AVX1-FAST-NEXT: vpmovsxbd %xmm3, %xmm3
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
; AVX1-FAST-NEXT: vpmovsxbd %xmm4, %xmm4
; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
; AVX1-FAST-NEXT: vpmovsxbd %xmm1, %xmm1
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
; AVX1-FAST-NEXT: vpmovsxbd %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm2, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: vzeroupper
@@ -1227,7 +1225,7 @@ define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
; AVX512-NEXT: vpmovsxbd %xmm0, %zmm0
; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1347,8 +1345,7 @@ define i16 @test_v4i16_v4i8(<4 x i8> %a0) nounwind {
; AVX1-FAST-LABEL: test_v4i16_v4i8:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vpmovsxbw %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1521,7 +1518,7 @@ define i16 @test_v16i16_v16i8(<16 x i8> %a0) nounwind {
; AVX1-FAST-NEXT: vpmovsxbw %xmm0, %xmm1
; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; AVX1-FAST-NEXT: vpmovsxbw %xmm0, %xmm0
-; AVX1-FAST-NEXT: vphaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm0, %xmm1, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
@@ -1647,10 +1644,8 @@ define i16 @test_v32i16_v32i8(<32 x i8> %a0) nounwind {
; AVX1-FAST-NEXT: vpmovsxbw %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1680,7 +1675,7 @@ define i16 @test_v32i16_v32i8(<32 x i8> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovsxbw %ymm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1706,25 +1701,25 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
; X86-SSE2-NEXT: psraw $8, %xmm4
-; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
; X86-SSE2-NEXT: psraw $8, %xmm5
; X86-SSE2-NEXT: paddw %xmm4, %xmm5
-; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm3[8],xmm6[9],xmm3[9],xmm6[10],xmm3[10],xmm6[11],xmm3[11],xmm6[12],xmm3[12],xmm6[13],xmm3[13],xmm6[14],xmm3[14],xmm6[15],xmm3[15]
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1],xmm6[2],xmm3[2],xmm6[3],xmm3[3],xmm6[4],xmm3[4],xmm6[5],xmm3[5],xmm6[6],xmm3[6],xmm6[7],xmm3[7]
; X86-SSE2-NEXT: psraw $8, %xmm6
-; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm1[8],xmm4[9],xmm1[9],xmm4[10],xmm1[10],xmm4[11],xmm1[11],xmm4[12],xmm1[12],xmm4[13],xmm1[13],xmm4[14],xmm1[14],xmm4[15],xmm1[15]
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
; X86-SSE2-NEXT: psraw $8, %xmm4
; X86-SSE2-NEXT: paddw %xmm6, %xmm4
; X86-SSE2-NEXT: paddw %xmm5, %xmm4
-; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X86-SSE2-NEXT: psraw $8, %xmm2
-; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X86-SSE2-NEXT: psraw $8, %xmm0
; X86-SSE2-NEXT: paddw %xmm2, %xmm0
-; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
; X86-SSE2-NEXT: psraw $8, %xmm2
-; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X86-SSE2-NEXT: psraw $8, %xmm1
; X86-SSE2-NEXT: paddw %xmm2, %xmm1
; X86-SSE2-NEXT: paddw %xmm0, %xmm1
@@ -1744,25 +1739,25 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
;
; X64-SSE2-LABEL: test_v64i16_v64i8:
; X64-SSE2: # %bb.0:
-; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
; X64-SSE2-NEXT: psraw $8, %xmm4
-; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm0[8],xmm5[9],xmm0[9],xmm5[10],xmm0[10],xmm5[11],xmm0[11],xmm5[12],xmm0[12],xmm5[13],xmm0[13],xmm5[14],xmm0[14],xmm5[15],xmm0[15]
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm0[0],xmm5[1],xmm0[1],xmm5[2],xmm0[2],xmm5[3],xmm0[3],xmm5[4],xmm0[4],xmm5[5],xmm0[5],xmm5[6],xmm0[6],xmm5[7],xmm0[7]
; X64-SSE2-NEXT: psraw $8, %xmm5
; X64-SSE2-NEXT: paddw %xmm4, %xmm5
-; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8],xmm3[8],xmm4[9],xmm3[9],xmm4[10],xmm3[10],xmm4[11],xmm3[11],xmm4[12],xmm3[12],xmm4[13],xmm3[13],xmm4[14],xmm3[14],xmm4[15],xmm3[15]
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3],xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
; X64-SSE2-NEXT: psraw $8, %xmm4
-; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm6 = xmm6[8],xmm1[8],xmm6[9],xmm1[9],xmm6[10],xmm1[10],xmm6[11],xmm1[11],xmm6[12],xmm1[12],xmm6[13],xmm1[13],xmm6[14],xmm1[14],xmm6[15],xmm1[15]
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm1[0],xmm6[1],xmm1[1],xmm6[2],xmm1[2],xmm6[3],xmm1[3],xmm6[4],xmm1[4],xmm6[5],xmm1[5],xmm6[6],xmm1[6],xmm6[7],xmm1[7]
; X64-SSE2-NEXT: psraw $8, %xmm6
; X64-SSE2-NEXT: paddw %xmm4, %xmm6
; X64-SSE2-NEXT: paddw %xmm5, %xmm6
-; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X64-SSE2-NEXT: psraw $8, %xmm2
-; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X64-SSE2-NEXT: psraw $8, %xmm0
; X64-SSE2-NEXT: paddw %xmm2, %xmm0
-; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3],xmm2[4],xmm3[4],xmm2[5],xmm3[5],xmm2[6],xmm3[6],xmm2[7],xmm3[7]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm3[8],xmm2[9],xmm3[9],xmm2[10],xmm3[10],xmm2[11],xmm3[11],xmm2[12],xmm3[12],xmm2[13],xmm3[13],xmm2[14],xmm3[14],xmm2[15],xmm3[15]
; X64-SSE2-NEXT: psraw $8, %xmm2
-; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
; X64-SSE2-NEXT: psraw $8, %xmm1
; X64-SSE2-NEXT: paddw %xmm2, %xmm1
; X64-SSE2-NEXT: paddw %xmm0, %xmm1
@@ -1784,26 +1779,26 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
; X86-SSE4-NEXT: movl %esp, %ebp
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm4
-; X86-SSE4-NEXT: pmovsxbw %xmm2, %xmm3
-; X86-SSE4-NEXT: pmovsxbw %xmm0, %xmm5
-; X86-SSE4-NEXT: paddw %xmm3, %xmm5
-; X86-SSE4-NEXT: pmovsxbw %xmm4, %xmm6
-; X86-SSE4-NEXT: pmovsxbw %xmm1, %xmm3
-; X86-SSE4-NEXT: paddw %xmm6, %xmm3
-; X86-SSE4-NEXT: paddw %xmm5, %xmm3
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
+; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm3
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; X86-SSE4-NEXT: pmovsxbw %xmm4, %xmm4
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: pmovsxbw %xmm5, %xmm5
+; X86-SSE4-NEXT: paddw %xmm4, %xmm5
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; X86-SSE4-NEXT: pmovsxbw %xmm4, %xmm4
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: pmovsxbw %xmm6, %xmm6
+; X86-SSE4-NEXT: paddw %xmm4, %xmm6
+; X86-SSE4-NEXT: paddw %xmm5, %xmm6
; X86-SSE4-NEXT: pmovsxbw %xmm2, %xmm2
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; X86-SSE4-NEXT: pmovsxbw %xmm0, %xmm0
; X86-SSE4-NEXT: paddw %xmm2, %xmm0
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
-; X86-SSE4-NEXT: pmovsxbw %xmm2, %xmm2
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: pmovsxbw %xmm3, %xmm2
; X86-SSE4-NEXT: pmovsxbw %xmm1, %xmm1
; X86-SSE4-NEXT: paddw %xmm2, %xmm1
; X86-SSE4-NEXT: paddw %xmm0, %xmm1
-; X86-SSE4-NEXT: paddw %xmm3, %xmm1
+; X86-SSE4-NEXT: paddw %xmm6, %xmm1
; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; X86-SSE4-NEXT: paddw %xmm1, %xmm0
; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1819,21 +1814,21 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
;
; X64-SSE4-LABEL: test_v64i16_v64i8:
; X64-SSE4: # %bb.0:
-; X64-SSE4-NEXT: pmovsxbw %xmm2, %xmm4
-; X64-SSE4-NEXT: pmovsxbw %xmm0, %xmm5
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; X64-SSE4-NEXT: pmovsxbw %xmm4, %xmm4
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: pmovsxbw %xmm5, %xmm5
; X64-SSE4-NEXT: paddw %xmm4, %xmm5
-; X64-SSE4-NEXT: pmovsxbw %xmm3, %xmm4
-; X64-SSE4-NEXT: pmovsxbw %xmm1, %xmm6
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; X64-SSE4-NEXT: pmovsxbw %xmm4, %xmm4
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: pmovsxbw %xmm6, %xmm6
; X64-SSE4-NEXT: paddw %xmm4, %xmm6
; X64-SSE4-NEXT: paddw %xmm5, %xmm6
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm2[2,3,2,3]
; X64-SSE4-NEXT: pmovsxbw %xmm2, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; X64-SSE4-NEXT: pmovsxbw %xmm0, %xmm0
; X64-SSE4-NEXT: paddw %xmm2, %xmm0
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
-; X64-SSE4-NEXT: pmovsxbw %xmm2, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: pmovsxbw %xmm3, %xmm2
; X64-SSE4-NEXT: pmovsxbw %xmm1, %xmm1
; X64-SSE4-NEXT: paddw %xmm2, %xmm1
; X64-SSE4-NEXT: paddw %xmm0, %xmm1
@@ -1851,27 +1846,27 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
;
; AVX1-SLOW-LABEL: test_v64i16_v64i8:
; AVX1-SLOW: # %bb.0:
-; AVX1-SLOW-NEXT: vpmovsxbw %xmm1, %xmm2
-; AVX1-SLOW-NEXT: vpmovsxbw %xmm0, %xmm3
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovsxbw %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovsxbw %xmm3, %xmm3
; AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm3, %xmm2
; AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm3
-; AVX1-SLOW-NEXT: vpmovsxbw %xmm3, %xmm4
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovsxbw %xmm4, %xmm4
; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm5
-; AVX1-SLOW-NEXT: vpmovsxbw %xmm5, %xmm6
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovsxbw %xmm6, %xmm6
; AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm6, %xmm4
; AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm2, %xmm2
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX1-SLOW-NEXT: vpmovsxbw %xmm1, %xmm1
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; AVX1-SLOW-NEXT: vpmovsxbw %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
-; AVX1-SLOW-NEXT: vpmovsxbw %xmm1, %xmm1
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[2,3,2,3]
-; AVX1-SLOW-NEXT: vpmovsxbw %xmm3, %xmm3
+; AVX1-SLOW-NEXT: vpmovsxbw %xmm3, %xmm1
+; AVX1-SLOW-NEXT: vpmovsxbw %xmm5, %xmm3
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm3, %xmm1
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-SLOW-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1885,31 +1880,29 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
;
; AVX1-FAST-LABEL: test_v64i16_v64i8:
; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vpmovsxbw %xmm1, %xmm2
-; AVX1-FAST-NEXT: vpmovsxbw %xmm0, %xmm3
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovsxbw %xmm2, %xmm2
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovsxbw %xmm3, %xmm3
; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm3, %xmm2
; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm3
-; AVX1-FAST-NEXT: vpmovsxbw %xmm3, %xmm4
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovsxbw %xmm4, %xmm4
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm5
-; AVX1-FAST-NEXT: vpmovsxbw %xmm5, %xmm6
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovsxbw %xmm6, %xmm6
; AVX1-FAST-NEXT: vpaddw %xmm4, %xmm6, %xmm4
; AVX1-FAST-NEXT: vpaddw %xmm4, %xmm2, %xmm2
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
; AVX1-FAST-NEXT: vpmovsxbw %xmm1, %xmm1
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; AVX1-FAST-NEXT: vpmovsxbw %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
-; AVX1-FAST-NEXT: vpmovsxbw %xmm1, %xmm1
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[2,3,2,3]
-; AVX1-FAST-NEXT: vpmovsxbw %xmm3, %xmm3
+; AVX1-FAST-NEXT: vpmovsxbw %xmm3, %xmm1
+; AVX1-FAST-NEXT: vpmovsxbw %xmm5, %xmm3
; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm3, %xmm1
; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpaddw %xmm0, %xmm2, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1947,7 +1940,7 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
; AVX512-NEXT: vpmovsxbw %ymm0, %zmm0
; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-subvector.ll b/llvm/test/CodeGen/X86/vector-reduce-add-subvector.ll
index f62b2a5cad2ed..db1d6e070f3b6 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-subvector.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-subvector.ll
@@ -11,8 +11,8 @@
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx,+fast-hops | FileCheck %s --check-prefixes=AVX,X64-AVX,AVX1,AVX1-FAST,X64-AVX1,X64-AVX1-FAST
; RUN: llc < %s -mtriple=i686-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,X86-AVX,AVX2,X86-AVX2
; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,X64-AVX,AVX2,X64-AVX2
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX,X64-AVX,AVX512
-; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX,X64-AVX,AVX512
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512bw | FileCheck %s --check-prefixes=AVX,X64-AVX,AVX512,AVX512F
+; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512f,+avx512bw,+avx512vl | FileCheck %s --check-prefixes=AVX,X64-AVX,AVX512,AVX512VL
;
; vXi64
@@ -349,17 +349,17 @@ define i32 @test_v4i32_v2i32(<4 x i32> %a0) nounwind {
define i32 @test_v4i32_v2i32_hi(<4 x i32> %a0) nounwind {
; SSE-LABEL: test_v4i32_v2i32_hi:
; SSE: # %bb.0:
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
; SSE-NEXT: paddd %xmm1, %xmm0
; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: ret{{[l|q]}}
;
; AVX1-SLOW-LABEL: test_v4i32_v2i32_hi:
; AVX1-SLOW: # %bb.0:
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm1, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
; AVX1-SLOW-NEXT: ret{{[l|q]}}
;
@@ -371,17 +371,17 @@ define i32 @test_v4i32_v2i32_hi(<4 x i32> %a0) nounwind {
;
; AVX2-LABEL: test_v4i32_v2i32_hi:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vpaddd %xmm0, %xmm1, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: ret{{[l|q]}}
;
; AVX512-LABEL: test_v4i32_v2i32_hi:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpaddd %xmm0, %xmm1, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: retq
%v = shufflevector <4 x i32> %a0, <4 x i32> poison, <2 x i32> <i32 2, i32 3>
@@ -611,7 +611,7 @@ define i32 @test_v16i32_v8i32(<16 x i32> %a0) nounwind {
; AVX1-FAST-LABEL: test_v16i32_v8i32:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
@@ -688,7 +688,7 @@ define i32 @test_v16i32_v8i32_mid(<16 x i32> %a0) nounwind {
; AVX1-FAST-LABEL: test_v16i32_v8i32_mid:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
@@ -709,9 +709,9 @@ define i32 @test_v16i32_v8i32_mid(<16 x i32> %a0) nounwind {
;
; AVX512-LABEL: test_v16i32_v8i32_mid:
; AVX512: # %bb.0:
-; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm0
-; AVX512-NEXT: vpaddd %xmm0, %xmm1, %xmm0
+; AVX512-NEXT: vextracti32x4 $2, %zmm0, %xmm1
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -731,21 +731,21 @@ define i32 @test_v16i32_v8i32_mid(<16 x i32> %a0) nounwind {
define i16 @test_v8i16_v4i16_hi(<8 x i16> %a0) nounwind {
; SSE-LABEL: test_v8i16_v4i16_hi:
; SSE: # %bb.0:
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; SSE-NEXT: paddw %xmm1, %xmm0
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrld $16, %xmm1
+; SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; SSE-NEXT: paddw %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
+; SSE-NEXT: movdqa %xmm1, %xmm0
+; SSE-NEXT: psrld $16, %xmm0
+; SSE-NEXT: paddw %xmm1, %xmm0
+; SSE-NEXT: movd %xmm0, %eax
; SSE-NEXT: # kill: def $ax killed $ax killed $eax
; SSE-NEXT: ret{{[l|q]}}
;
; AVX1-SLOW-LABEL: test_v8i16_v4i16_hi:
; AVX1-SLOW: # %bb.0:
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX1-SLOW-NEXT: vpaddw %xmm0, %xmm1, %xmm0
+; AVX1-SLOW-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
@@ -754,9 +754,9 @@ define i16 @test_v8i16_v4i16_hi(<8 x i16> %a0) nounwind {
;
; AVX1-FAST-LABEL: test_v8i16_v4i16_hi:
; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX1-FAST-NEXT: vpaddw %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vphaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,3,1,3]
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -764,25 +764,36 @@ define i16 @test_v8i16_v4i16_hi(<8 x i16> %a0) nounwind {
;
; AVX2-LABEL: test_v8i16_v4i16_hi:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX2-NEXT: vpaddw %xmm0, %xmm1, %xmm0
+; AVX2-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: ret{{[l|q]}}
;
-; AVX512-LABEL: test_v8i16_v4i16_hi:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
-; AVX512-NEXT: vpaddw %xmm0, %xmm1, %xmm0
-; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: retq
+; AVX512F-LABEL: test_v8i16_v4i16_hi:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512F-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX512F-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vmovd %xmm0, %eax
+; AVX512F-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: test_v8i16_v4i16_hi:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
+; AVX512VL-NEXT: vshufps {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512VL-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX512VL-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovd %xmm0, %eax
+; AVX512VL-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512VL-NEXT: retq
%v = shufflevector <8 x i16> %a0, <8 x i16> poison, <4 x i32> <i32 4, i32 5, i32 6, i32 7>
%r = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %v)
ret i16 %r
@@ -912,7 +923,7 @@ define i16 @test_v32i16_v16i16(<32 x i16> %a0) nounwind {
; AVX1-FAST-LABEL: test_v32i16_v16i16:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
@@ -962,7 +973,7 @@ define i16 @test_v32i16_v16i16(<32 x i16> %a0) nounwind {
define i8 @test_v16i8_v8i8_hi(<16 x i8> %a0) nounwind {
; SSE-LABEL: test_v16i8_v8i8_hi:
; SSE: # %bb.0:
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; SSE-NEXT: psrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
; SSE-NEXT: pxor %xmm1, %xmm1
; SSE-NEXT: psadbw %xmm0, %xmm1
; SSE-NEXT: movd %xmm1, %eax
@@ -971,7 +982,7 @@ define i8 @test_v16i8_v8i8_hi(<16 x i8> %a0) nounwind {
;
; AVX-LABEL: test_v16i8_v8i8_hi:
; AVX: # %bb.0:
-; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero
; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX-NEXT: vmovd %xmm0, %eax
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll b/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
index 9d13de751812f..8f14903dc43db 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add-zext.ll
@@ -219,61 +219,159 @@ define i64 @test_v8i64_v8i8(<8 x i8> %a0) nounwind {
; X86-SSE2-LABEL: test_v8i64_v8i8:
; X86-SSE2: # %bb.0:
; X86-SSE2-NEXT: pxor %xmm1, %xmm1
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm4
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; X86-SSE2-NEXT: paddq %xmm3, %xmm4
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-SSE2-NEXT: paddq %xmm2, %xmm0
+; X86-SSE2-NEXT: paddq %xmm4, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE2-NEXT: paddq %xmm0, %xmm1
; X86-SSE2-NEXT: movd %xmm1, %eax
-; X86-SSE2-NEXT: xorl %edx, %edx
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X86-SSE2-NEXT: movd %xmm0, %edx
; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v8i64_v8i8:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pxor %xmm1, %xmm1
-; X64-SSE-NEXT: psadbw %xmm0, %xmm1
-; X64-SSE-NEXT: movq %xmm1, %rax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v8i64_v8i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm2
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm4
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1]
+; X64-SSE2-NEXT: paddq %xmm3, %xmm4
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X64-SSE2-NEXT: paddq %xmm2, %xmm0
+; X64-SSE2-NEXT: paddq %xmm4, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE2-NEXT: paddq %xmm0, %xmm1
+; X64-SSE2-NEXT: movq %xmm1, %rax
+; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v8i64_v8i8:
; X86-SSE4: # %bb.0:
-; X86-SSE4-NEXT: pxor %xmm1, %xmm1
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: paddq %xmm1, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE4-NEXT: psrlq $48, %xmm1
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: psrld $16, %xmm0
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: paddq %xmm1, %xmm0
+; X86-SSE4-NEXT: paddq %xmm2, %xmm0
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm0, %xmm1
; X86-SSE4-NEXT: movd %xmm1, %eax
; X86-SSE4-NEXT: pextrd $1, %xmm1, %edx
; X86-SSE4-NEXT: retl
;
+; X64-SSE4-LABEL: test_v8i64_v8i8:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: paddq %xmm1, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE4-NEXT: psrlq $48, %xmm1
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: psrld $16, %xmm0
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: paddq %xmm1, %xmm0
+; X64-SSE4-NEXT: paddq %xmm2, %xmm0
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm0, %xmm1
+; X64-SSE4-NEXT: movq %xmm1, %rax
+; X64-SSE4-NEXT: retq
+;
; X86-AVX1-LABEL: test_v8i64_v8i8:
; X86-AVX1: # %bb.0:
-; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsrld $16, %xmm0, %xmm1
+; X86-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; X86-AVX1-NEXT: vpsrld $16, %xmm2, %xmm3
+; X86-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: vpextrd $1, %xmm0, %edx
; X86-AVX1-NEXT: retl
;
; X64-AVX1-LABEL: test_v8i64_v8i8:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsrld $16, %xmm0, %xmm1
+; X64-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; X64-AVX1-NEXT: vpsrld $16, %xmm2, %xmm3
+; X64-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: test_v8i64_v8i8:
; X86-AVX2: # %bb.0:
-; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-AVX2-NEXT: vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm0
+; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: vpextrd $1, %xmm0, %edx
+; X86-AVX2-NEXT: vzeroupper
; X86-AVX2-NEXT: retl
;
; X64-AVX2-LABEL: test_v8i64_v8i8:
; X64-AVX2: # %bb.0:
-; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpmovzxbq {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X64-AVX2-NEXT: vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm0
+; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
; AVX512-LABEL: test_v8i64_v8i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpmovzxbq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,xmm0[4],zero,zero,zero,zero,zero,zero,zero,xmm0[5],zero,zero,zero,zero,zero,zero,zero,xmm0[6],zero,zero,zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero,zero,zero
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%1 = zext <8 x i8> %a0 to <8 x i64>
%2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %1)
@@ -283,38 +381,169 @@ define i64 @test_v8i64_v8i8(<8 x i8> %a0) nounwind {
define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
; X86-SSE2-LABEL: test_v16i64_v16i8:
; X86-SSE2: # %bb.0:
-; X86-SSE2-NEXT: pxor %xmm1, %xmm1
-; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE2-NEXT: pxor %xmm2, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm5
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm6
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
+; X86-SSE2-NEXT: paddq %xmm5, %xmm6
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm7
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1]
+; X86-SSE2-NEXT: paddq %xmm7, %xmm5
+; X86-SSE2-NEXT: paddq %xmm6, %xmm5
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm4 = xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X86-SSE2-NEXT: paddq %xmm4, %xmm3
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
; X86-SSE2-NEXT: paddq %xmm1, %xmm0
-; X86-SSE2-NEXT: movd %xmm0, %eax
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT: paddq %xmm3, %xmm0
+; X86-SSE2-NEXT: paddq %xmm5, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE2-NEXT: paddq %xmm0, %xmm1
+; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; X86-SSE2-NEXT: movd %xmm0, %edx
; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v16i64_v16i8:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pxor %xmm1, %xmm1
-; X64-SSE-NEXT: psadbw %xmm0, %xmm1
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; X64-SSE-NEXT: paddq %xmm1, %xmm0
-; X64-SSE-NEXT: movq %xmm0, %rax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v16i64_v16i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pxor %xmm2, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm4
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm5
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm6
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1]
+; X64-SSE2-NEXT: paddq %xmm4, %xmm6
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm4
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm7
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1]
+; X64-SSE2-NEXT: paddq %xmm4, %xmm7
+; X64-SSE2-NEXT: paddq %xmm6, %xmm7
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm3 = xmm3[2],xmm2[2],xmm3[3],xmm2[3]
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm5 = xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X64-SSE2-NEXT: paddq %xmm3, %xmm5
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3]
+; X64-SSE2-NEXT: paddq %xmm1, %xmm0
+; X64-SSE2-NEXT: paddq %xmm5, %xmm0
+; X64-SSE2-NEXT: paddq %xmm7, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE2-NEXT: paddq %xmm0, %xmm1
+; X64-SSE2-NEXT: movq %xmm1, %rax
+; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v16i64_v16i8:
; X86-SSE4: # %bb.0:
-; X86-SSE4-NEXT: pxor %xmm1, %xmm1
-; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: paddq %xmm1, %xmm2
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: paddq %xmm3, %xmm1
+; X86-SSE4-NEXT: paddq %xmm2, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE4-NEXT: psrldq {{.*#+}} xmm2 = xmm2[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE4-NEXT: psrld $16, %xmm3
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: paddq %xmm2, %xmm3
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE4-NEXT: psrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: psrlq $48, %xmm0
+; X86-SSE4-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X86-SSE4-NEXT: paddq %xmm2, %xmm0
+; X86-SSE4-NEXT: paddq %xmm3, %xmm0
; X86-SSE4-NEXT: paddq %xmm1, %xmm0
-; X86-SSE4-NEXT: movd %xmm0, %eax
-; X86-SSE4-NEXT: pextrd $1, %xmm0, %edx
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm0, %xmm1
+; X86-SSE4-NEXT: movd %xmm1, %eax
+; X86-SSE4-NEXT: pextrd $1, %xmm1, %edx
; X86-SSE4-NEXT: retl
;
+; X64-SSE4-LABEL: test_v16i64_v16i8:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: paddq %xmm1, %xmm2
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: paddq %xmm1, %xmm3
+; X64-SSE4-NEXT: paddq %xmm2, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE4-NEXT: psrldq {{.*#+}} xmm1 = xmm1[10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X64-SSE4-NEXT: psrld $16, %xmm2
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: paddq %xmm1, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE4-NEXT: psrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,zero,zero,zero,zero,xmm1[1],zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: psrlq $48, %xmm0
+; X64-SSE4-NEXT: pmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X64-SSE4-NEXT: paddq %xmm1, %xmm0
+; X64-SSE4-NEXT: paddq %xmm2, %xmm0
+; X64-SSE4-NEXT: paddq %xmm3, %xmm0
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm0, %xmm1
+; X64-SSE4-NEXT: movq %xmm1, %rax
+; X64-SSE4-NEXT: retq
+;
; X86-AVX1-LABEL: test_v16i64_v16i8:
; X86-AVX1: # %bb.0:
; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X86-AVX1-NEXT: vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; X86-AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; X86-AVX1-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; X86-AVX1-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; X86-AVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
+; X86-AVX1-NEXT: vpmovzxwq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero
+; X86-AVX1-NEXT: vpaddq %xmm2, %xmm6, %xmm2
+; X86-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm6 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT: vpaddq %xmm3, %xmm6, %xmm3
+; X86-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
+; X86-AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX1-NEXT: vpaddq %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[3,3,3,3]
+; X86-AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm3, %xmm1
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
@@ -324,7 +553,29 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
; X64-AVX1-LABEL: test_v16i64_v16i8:
; X64-AVX1: # %bb.0:
; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X64-AVX1-NEXT: vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero
+; X64-AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; X64-AVX1-NEXT: vpmovzxwq {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; X64-AVX1-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero
+; X64-AVX1-NEXT: vpmovzxbw {{.*#+}} xmm5 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm5[2,3,2,3]
+; X64-AVX1-NEXT: vpmovzxwq {{.*#+}} xmm6 = xmm6[0],zero,zero,zero,xmm6[1],zero,zero,zero
+; X64-AVX1-NEXT: vpaddq %xmm2, %xmm6, %xmm2
+; X64-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm6 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX1-NEXT: vpaddq %xmm3, %xmm6, %xmm3
+; X64-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
+; X64-AVX1-NEXT: vpsrld $16, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX1-NEXT: vpaddq %xmm4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[3,3,3,3]
+; X64-AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm3, %xmm1
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
@@ -332,30 +583,62 @@ define i64 @test_v16i64_v16i8(<16 x i8> %a0) nounwind {
;
; X86-AVX2-LABEL: test_v16i64_v16i8:
; X86-AVX2: # %bb.0:
-; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; X86-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; X86-AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X86-AVX2-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X86-AVX2-NEXT: vpaddq %ymm3, %ymm1, %ymm1
+; X86-AVX2-NEXT: vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; X86-AVX2-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; X86-AVX2-NEXT: vpaddq %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: vpextrd $1, %xmm0, %edx
+; X86-AVX2-NEXT: vzeroupper
; X86-AVX2-NEXT: retl
;
; X64-AVX2-LABEL: test_v16i64_v16i8:
; X64-AVX2: # %bb.0:
-; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; X64-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; X64-AVX2-NEXT: vpmovzxwq {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; X64-AVX2-NEXT: vpmovzxwq {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X64-AVX2-NEXT: vpaddq %ymm3, %ymm1, %ymm1
+; X64-AVX2-NEXT: vpmovzxbq {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero
+; X64-AVX2-NEXT: vpmovzxwq {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; X64-AVX2-NEXT: vpaddq %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
; AVX512-LABEL: test_v16i64_v16i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX512-NEXT: vpmovzxwq {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%1 = zext <16 x i8> %a0 to <16 x i64>
%2 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %1)
@@ -421,45 +704,101 @@ define i32 @test_v2i32_v2i16(<2 x i16> %a0) nounwind {
}
define i32 @test_v4i32(<4 x i8> %a0) nounwind {
-; SSE2-LABEL: test_v4i32:
-; SSE2: # %bb.0:
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: psadbw %xmm1, %xmm0
-; SSE2-NEXT: movd %xmm0, %eax
-; SSE2-NEXT: ret{{[l|q]}}
+; X86-SSE2-LABEL: test_v4i32:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE2-NEXT: paddq %xmm0, %xmm1
+; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: retl
;
-; SSE4-LABEL: test_v4i32:
-; SSE4: # %bb.0:
-; SSE4-NEXT: pxor %xmm1, %xmm1
-; SSE4-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
-; SSE4-NEXT: psadbw %xmm1, %xmm0
-; SSE4-NEXT: movd %xmm0, %eax
-; SSE4-NEXT: ret{{[l|q]}}
+; X64-SSE2-LABEL: test_v4i32:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE2-NEXT: paddq %xmm0, %xmm1
+; X64-SSE2-NEXT: movq %xmm1, %rax
+; X64-SSE2-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-SSE2-NEXT: retq
;
-; AVX1-LABEL: test_v4i32:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
-; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: ret{{[l|q]}}
+; X86-SSE4-LABEL: test_v4i32:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: retl
;
-; AVX2-LABEL: test_v4i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
-; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: ret{{[l|q]}}
+; X64-SSE4-LABEL: test_v4i32:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm1, %xmm0
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-SSE4-NEXT: retq
+;
+; X86-AVX1-LABEL: test_v4i32:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v4i32:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX1-NEXT: retq
+;
+; X86-AVX2-LABEL: test_v4i32:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: test_v4i32:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX2-NEXT: retq
;
; AVX512-LABEL: test_v4i32:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: retq
%1 = zext <4 x i8> %a0 to <4 x i32>
%2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %1)
@@ -467,159 +806,549 @@ define i32 @test_v4i32(<4 x i8> %a0) nounwind {
}
define i32 @test_v8i32_v8i8(<8 x i8> %a0) nounwind {
-; SSE-LABEL: test_v8i32_v8i8:
-; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm1, %xmm1
-; SSE-NEXT: psadbw %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX-LABEL: test_v8i32_v8i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: ret{{[l|q]}}
- %1 = zext <8 x i8> %a0 to <8 x i32>
- %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
- ret i32 %2
-}
-
-define i32 @test_v16i32_v16i8(<16 x i8> %a0) nounwind {
-; SSE-LABEL: test_v16i32_v16i8:
-; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm1, %xmm1
-; SSE-NEXT: psadbw %xmm0, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; SSE-NEXT: paddq %xmm1, %xmm0
-; SSE-NEXT: movd %xmm0, %eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX-LABEL: test_v16i32_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: ret{{[l|q]}}
- %1 = zext <16 x i8> %a0 to <16 x i32>
- %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)
- ret i32 %2
-}
-
-define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
-; SSE-LABEL: test_v32i32_v32i8:
-; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm2, %xmm2
-; SSE-NEXT: psadbw %xmm2, %xmm1
-; SSE-NEXT: psadbw %xmm2, %xmm0
-; SSE-NEXT: paddq %xmm1, %xmm0
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE-NEXT: paddq %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX1-LABEL: test_v32i32_v32i8:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: vzeroupper
-; AVX1-NEXT: ret{{[l|q]}}
+; SSE2-LABEL: test_v8i32_v8i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: ret{{[l|q]}}
;
-; AVX2-LABEL: test_v32i32_v32i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: ret{{[l|q]}}
+; SSE4-LABEL: test_v8i32_v8i8:
+; SSE4: # %bb.0:
+; SSE4-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE4-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE4-NEXT: paddd %xmm1, %xmm0
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE4-NEXT: paddd %xmm0, %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE4-NEXT: paddd %xmm1, %xmm0
+; SSE4-NEXT: movd %xmm0, %eax
+; SSE4-NEXT: ret{{[l|q]}}
;
-; AVX512-LABEL: test_v32i32_v32i8:
+; X86-AVX1-LABEL: test_v8i32_v8i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X86-AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v8i32_v8i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X64-AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX1-NEXT: retq
+;
+; X86-AVX2-LABEL: test_v8i32_v8i8:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: vzeroupper
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: test_v8i32_v8i8:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX2-NEXT: vzeroupper
+; X64-AVX2-NEXT: retq
+;
+; AVX512-LABEL: test_v8i32_v8i8:
; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
- %1 = zext <32 x i8> %a0 to <32 x i32>
- %2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)
+ %1 = zext <8 x i8> %a0 to <8 x i32>
+ %2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %1)
ret i32 %2
}
-;
-; vXi16
-;
-
-define i16 @test_v2i16_v2i8(<2 x i8> %a0) nounwind {
-; SSE2-LABEL: test_v2i16_v2i8:
+define i32 @test_v16i32_v16i8(<16 x i8> %a0) nounwind {
+; SSE2-LABEL: test_v16i32_v16i8:
; SSE2: # %bb.0:
; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
+; SSE2-NEXT: movdqa %xmm2, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3]
; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $16, %xmm1
-; SSE2-NEXT: paddw %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
-; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSE2-NEXT: paddd %xmm3, %xmm4
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm2 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: paddd %xmm4, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: ret{{[l|q]}}
;
-; SSE4-LABEL: test_v2i16_v2i8:
+; SSE4-LABEL: test_v16i32_v16i8:
; SSE4: # %bb.0:
-; SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; SSE4-NEXT: movdqa %xmm0, %xmm1
-; SSE4-NEXT: psrld $16, %xmm1
-; SSE4-NEXT: paddw %xmm0, %xmm1
-; SSE4-NEXT: movd %xmm1, %eax
-; SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; SSE4-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; SSE4-NEXT: paddd %xmm1, %xmm2
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE4-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE4-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; SSE4-NEXT: paddd %xmm1, %xmm0
+; SSE4-NEXT: paddd %xmm2, %xmm0
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE4-NEXT: paddd %xmm0, %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE4-NEXT: paddd %xmm1, %xmm0
+; SSE4-NEXT: movd %xmm0, %eax
; SSE4-NEXT: ret{{[l|q]}}
;
-; AVX1-SLOW-LABEL: test_v2i16_v2i8:
+; AVX1-SLOW-LABEL: test_v16i32_v16i8:
; AVX1-SLOW: # %bb.0:
-; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm1, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
-; AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
; AVX1-SLOW-NEXT: ret{{[l|q]}}
;
-; AVX1-FAST-LABEL: test_v2i16_v2i8:
+; AVX1-FAST-LABEL: test_v16i32_v16i8:
; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
-; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
; AVX1-FAST-NEXT: ret{{[l|q]}}
;
-; AVX2-LABEL: test_v2i16_v2i8:
+; AVX2-LABEL: test_v16i32_v16i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: ret{{[l|q]}}
;
+; AVX512-LABEL: test_v16i32_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $eax killed $eax killed $rax
+; AVX512-NEXT: retq
+ %1 = zext <16 x i8> %a0 to <16 x i32>
+ %2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %1)
+ ret i32 %2
+}
+
+define i32 @test_v32i32_v32i8(<32 x i8> %a0) nounwind {
+; X86-SSE2-LABEL: test_v32i32_v32i8:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pxor %xmm2, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm5
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm6
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; X86-SSE2-NEXT: paddd %xmm5, %xmm6
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm7
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
+; X86-SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3]
+; X86-SSE2-NEXT: paddd %xmm7, %xmm5
+; X86-SSE2-NEXT: paddd %xmm6, %xmm5
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X86-SSE2-NEXT: paddd %xmm4, %xmm3
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; X86-SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; X86-SSE2-NEXT: paddd %xmm1, %xmm0
+; X86-SSE2-NEXT: paddd %xmm3, %xmm0
+; X86-SSE2-NEXT: paddd %xmm5, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE2-NEXT: paddd %xmm0, %xmm1
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X86-SSE2-NEXT: paddd %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: retl
+;
+; X64-SSE2-LABEL: test_v32i32_v32i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pxor %xmm2, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm4
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm5
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm2[0],xmm5[1],xmm2[1],xmm5[2],xmm2[2],xmm5[3],xmm2[3],xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm6
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; X64-SSE2-NEXT: paddd %xmm4, %xmm6
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm4
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm7
+; X64-SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; X64-SSE2-NEXT: paddd %xmm4, %xmm7
+; X64-SSE2-NEXT: paddd %xmm6, %xmm7
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; X64-SSE2-NEXT: paddd %xmm3, %xmm5
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm2[4],xmm1[5],xmm2[5],xmm1[6],xmm2[6],xmm1[7],xmm2[7]
+; X64-SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7]
+; X64-SSE2-NEXT: paddd %xmm1, %xmm0
+; X64-SSE2-NEXT: paddd %xmm5, %xmm0
+; X64-SSE2-NEXT: paddd %xmm7, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE2-NEXT: paddd %xmm0, %xmm1
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X64-SSE2-NEXT: paddd %xmm1, %xmm0
+; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: test_v32i32_v32i8:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; X86-SSE4-NEXT: paddd %xmm2, %xmm3
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-SSE4-NEXT: paddd %xmm4, %xmm2
+; X86-SSE4-NEXT: paddd %xmm3, %xmm2
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; X86-SSE4-NEXT: paddd %xmm3, %xmm4
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; X86-SSE4-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X86-SSE4-NEXT: paddd %xmm1, %xmm0
+; X86-SSE4-NEXT: paddd %xmm4, %xmm0
+; X86-SSE4-NEXT: paddd %xmm2, %xmm0
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: paddd %xmm0, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X86-SSE4-NEXT: paddd %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: test_v32i32_v32i8:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; X64-SSE4-NEXT: paddd %xmm2, %xmm3
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-SSE4-NEXT: paddd %xmm2, %xmm4
+; X64-SSE4-NEXT: paddd %xmm3, %xmm4
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; X64-SSE4-NEXT: paddd %xmm2, %xmm3
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; X64-SSE4-NEXT: pmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; X64-SSE4-NEXT: paddd %xmm1, %xmm0
+; X64-SSE4-NEXT: paddd %xmm3, %xmm0
+; X64-SSE4-NEXT: paddd %xmm4, %xmm0
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE4-NEXT: paddd %xmm0, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X64-SSE4-NEXT: paddd %xmm1, %xmm0
+; X64-SSE4-NEXT: movd %xmm0, %eax
+; X64-SSE4-NEXT: retq
+;
+; AVX1-SLOW-LABEL: test_v32i32_v32i8:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-SLOW-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm2, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT: vzeroupper
+; AVX1-SLOW-NEXT: ret{{[l|q]}}
+;
+; AVX1-FAST-LABEL: test_v32i32_v32i8:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm4 = xmm4[0],zero,zero,zero,xmm4[1],zero,zero,zero,xmm4[2],zero,zero,zero,xmm4[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-FAST-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm3, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm2, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; AVX1-FAST-NEXT: vzeroupper
+; AVX1-FAST-NEXT: ret{{[l|q]}}
+;
+; AVX2-LABEL: test_v32i32_v32i8:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero,xmm2[4],zero,zero,zero,xmm2[5],zero,zero,zero,xmm2[6],zero,zero,zero,xmm2[7],zero,zero,zero
+; AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero,xmm3[4],zero,zero,zero,xmm3[5],zero,zero,zero,xmm3[6],zero,zero,zero,xmm3[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: ret{{[l|q]}}
+;
+; AVX512-LABEL: test_v32i32_v32i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpmovzxbd {{.*#+}} zmm1 = xmm1[0],zero,zero,zero,xmm1[1],zero,zero,zero,xmm1[2],zero,zero,zero,xmm1[3],zero,zero,zero,xmm1[4],zero,zero,zero,xmm1[5],zero,zero,zero,xmm1[6],zero,zero,zero,xmm1[7],zero,zero,zero,xmm1[8],zero,zero,zero,xmm1[9],zero,zero,zero,xmm1[10],zero,zero,zero,xmm1[11],zero,zero,zero,xmm1[12],zero,zero,zero,xmm1[13],zero,zero,zero,xmm1[14],zero,zero,zero,xmm1[15],zero,zero,zero
+; AVX512-NEXT: vpmovzxbd {{.*#+}} zmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero,xmm0[4],zero,zero,zero,xmm0[5],zero,zero,zero,xmm0[6],zero,zero,zero,xmm0[7],zero,zero,zero,xmm0[8],zero,zero,zero,xmm0[9],zero,zero,zero,xmm0[10],zero,zero,zero,xmm0[11],zero,zero,zero,xmm0[12],zero,zero,zero,xmm0[13],zero,zero,zero,xmm0[14],zero,zero,zero,xmm0[15],zero,zero,zero
+; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
+ %1 = zext <32 x i8> %a0 to <32 x i32>
+ %2 = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %1)
+ ret i32 %2
+}
+
+;
+; vXi16
+;
+
+define i16 @test_v2i16_v2i8(<2 x i8> %a0) nounwind {
+; X86-SSE2-LABEL: test_v2i16_v2i8:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-SSE2-NEXT: xorps %xmm2, %xmm2
+; X86-SSE2-NEXT: movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm2
+; X86-SSE2-NEXT: movd %xmm2, %eax
+; X86-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT: retl
+;
+; X64-SSE2-LABEL: test_v2i16_v2i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X64-SSE2-NEXT: xorps %xmm2, %xmm2
+; X64-SSE2-NEXT: movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm2
+; X64-SSE2-NEXT: movq %xmm2, %rax
+; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: test_v2i16_v2i8:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: test_v2i16_v2i8:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-SSE4-NEXT: retq
+;
+; X86-AVX1-LABEL: test_v2i16_v2i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v2i16_v2i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7]
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX1-NEXT: retq
+;
+; X86-AVX2-LABEL: test_v2i16_v2i8:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: test_v2i16_v2i8:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX2-NEXT: retq
+;
; AVX512-LABEL: test_v2i16_v2i8:
; AVX512: # %bb.0:
; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3]
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $rax
; AVX512-NEXT: retq
%1 = zext <2 x i8> %a0 to <2 x i16>
%2 = call i16 @llvm.vector.reduce.add.v2i16(<2 x i16> %1)
@@ -627,22 +1356,330 @@ define i16 @test_v2i16_v2i8(<2 x i8> %a0) nounwind {
}
define i16 @test_v4i16_v4i8(<4 x i8> %a0) nounwind {
-; SSE2-LABEL: test_v4i16_v4i8:
+; X86-SSE2-LABEL: test_v4i16_v4i8:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT: retl
+;
+; X64-SSE2-LABEL: test_v4i16_v4i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: test_v4i16_v4i8:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE4-NEXT: movd %xmm1, %eax
+; X86-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: test_v4i16_v4i8:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE4-NEXT: movq %xmm1, %rax
+; X64-SSE4-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-SSE4-NEXT: retq
+;
+; X86-AVX1-LABEL: test_v4i16_v4i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v4i16_v4i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX1-NEXT: retq
+;
+; X86-AVX2-LABEL: test_v4i16_v4i8:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: test_v4i16_v4i8:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX2-NEXT: retq
+;
+; AVX512-LABEL: test_v4i16_v4i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX512-NEXT: retq
+ %1 = zext <4 x i8> %a0 to <4 x i16>
+ %2 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %1)
+ ret i16 %2
+
+}
+
+define i16 @test_v8i16_v8i8(<8 x i8> %a0) nounwind {
+; X86-SSE2-LABEL: test_v8i16_v8i8:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X86-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE2-NEXT: paddq %xmm0, %xmm1
+; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT: retl
+;
+; X64-SSE2-LABEL: test_v8i16_v8i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
+; X64-SSE2-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE2-NEXT: paddq %xmm0, %xmm1
+; X64-SSE2-NEXT: movq %xmm1, %rax
+; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: test_v8i16_v8i8:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: test_v8i16_v8i8:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm1, %xmm0
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-SSE4-NEXT: retq
+;
+; X86-AVX1-LABEL: test_v8i16_v8i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v8i16_v8i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX1-NEXT: retq
+;
+; X86-AVX2-LABEL: test_v8i16_v8i8:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: test_v8i16_v8i8:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX2-NEXT: retq
+;
+; AVX512-LABEL: test_v8i16_v8i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX512-NEXT: retq
+ %1 = zext <8 x i8> %a0 to <8 x i16>
+ %2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %1)
+ ret i16 %2
+}
+
+define i16 @test_v16i16_v16i8(<16 x i8> %a0) nounwind {
+; SSE2-LABEL: test_v16i16_v16i8:
; SSE2: # %bb.0:
; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm1[8],xmm2[9],xmm1[9],xmm2[10],xmm1[10],xmm2[11],xmm1[11],xmm2[12],xmm1[12],xmm2[13],xmm1[13],xmm2[14],xmm1[14],xmm2[15],xmm1[15]
; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7]
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddw %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddw %xmm0, %xmm1
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; SSE2-NEXT: paddw %xmm1, %xmm0
-; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: psrld $16, %xmm1
+; SSE2-NEXT: paddw %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: ret{{[l|q]}}
;
-; SSE4-LABEL: test_v4i16_v4i8:
+; SSE4-LABEL: test_v16i16_v16i8:
; SSE4: # %bb.0:
-; SSE4-NEXT: pmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE4-NEXT: pxor %xmm1, %xmm1
+; SSE4-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE4-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; SSE4-NEXT: paddw %xmm2, %xmm0
+; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE4-NEXT: paddw %xmm0, %xmm1
+; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE4-NEXT: paddw %xmm1, %xmm0
+; SSE4-NEXT: movdqa %xmm0, %xmm1
+; SSE4-NEXT: psrld $16, %xmm1
+; SSE4-NEXT: paddw %xmm0, %xmm1
+; SSE4-NEXT: movd %xmm1, %eax
+; SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE4-NEXT: ret{{[l|q]}}
+;
+; X86-AVX1-LABEL: test_v16i16_v16i8:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; X86-AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-NEXT: retl
+;
+; X64-AVX1-LABEL: test_v16i16_v16i8:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm1[8],xmm0[9],xmm1[9],xmm0[10],xmm1[10],xmm0[11],xmm1[11],xmm0[12],xmm1[12],xmm0[13],xmm1[13],xmm0[14],xmm1[14],xmm0[15],xmm1[15]
+; X64-AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX1-NEXT: retq
+;
+; X86-AVX2-LABEL: test_v16i16_v16i8:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: test_v16i16_v16i8:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX2-NEXT: retq
+;
+; AVX512-LABEL: test_v16i16_v16i8:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX512-NEXT: retq
+ %1 = zext <16 x i8> %a0 to <16 x i16>
+ %2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %1)
+ ret i16 %2
+}
+
+define i16 @test_v32i16_v32i8(<32 x i8> %a0) nounwind {
+; SSE2-LABEL: test_v32i16_v32i8:
+; SSE2: # %bb.0:
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1],xmm3[2],xmm2[2],xmm3[3],xmm2[3],xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm0, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1],xmm4[2],xmm2[2],xmm4[3],xmm2[3],xmm4[4],xmm2[4],xmm4[5],xmm2[5],xmm4[6],xmm2[6],xmm4[7],xmm2[7]
+; SSE2-NEXT: paddw %xmm3, %xmm4
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; SSE2-NEXT: paddw %xmm1, %xmm0
+; SSE2-NEXT: paddw %xmm4, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddw %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddw %xmm1, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: psrld $16, %xmm1
+; SSE2-NEXT: paddw %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: ret{{[l|q]}}
+;
+; SSE4-LABEL: test_v32i16_v32i8:
+; SSE4: # %bb.0:
+; SSE4-NEXT: pxor %xmm2, %xmm2
+; SSE4-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; SSE4-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; SSE4-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; SSE4-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; SSE4-NEXT: paddw %xmm1, %xmm0
+; SSE4-NEXT: paddw %xmm3, %xmm4
+; SSE4-NEXT: paddw %xmm0, %xmm4
+; SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm4[2,3,2,3]
+; SSE4-NEXT: paddw %xmm4, %xmm0
; SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; SSE4-NEXT: paddw %xmm0, %xmm1
; SSE4-NEXT: movdqa %xmm1, %xmm0
@@ -652,137 +1689,63 @@ define i16 @test_v4i16_v4i8(<4 x i8> %a0) nounwind {
; SSE4-NEXT: # kill: def $ax killed $ax killed $eax
; SSE4-NEXT: ret{{[l|q]}}
;
-; AVX1-SLOW-LABEL: test_v4i16_v4i8:
+; AVX1-SLOW-LABEL: test_v32i16_v32i8:
; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; AVX1-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
; AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-SLOW-NEXT: vzeroupper
; AVX1-SLOW-NEXT: ret{{[l|q]}}
;
-; AVX1-FAST-LABEL: test_v4i16_v4i8:
+; AVX1-FAST-LABEL: test_v32i16_v32i8:
; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
+; AVX1-FAST-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-FAST-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; AVX1-FAST-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX1-FAST-NEXT: vpaddw %xmm3, %xmm2, %xmm2
+; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-FAST-NEXT: vzeroupper
; AVX1-FAST-NEXT: ret{{[l|q]}}
;
-; AVX2-LABEL: test_v4i16_v4i8:
+; AVX2-LABEL: test_v32i16_v32i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: ret{{[l|q]}}
-;
-; AVX512-LABEL: test_v4i16_v4i8:
-; AVX512: # %bb.0:
-; AVX512-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: retq
- %1 = zext <4 x i8> %a0 to <4 x i16>
- %2 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %1)
- ret i16 %2
-
-}
-
-define i16 @test_v8i16_v8i8(<8 x i8> %a0) nounwind {
-; SSE-LABEL: test_v8i16_v8i8:
-; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm1, %xmm1
-; SSE-NEXT: psadbw %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX-LABEL: test_v8i16_v8i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: ret{{[l|q]}}
- %1 = zext <8 x i8> %a0 to <8 x i16>
- %2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %1)
- ret i16 %2
-}
-
-define i16 @test_v16i16_v16i8(<16 x i8> %a0) nounwind {
-; SSE-LABEL: test_v16i16_v16i8:
-; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm1, %xmm1
-; SSE-NEXT: psadbw %xmm0, %xmm1
-; SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; SSE-NEXT: paddq %xmm1, %xmm0
-; SSE-NEXT: movd %xmm0, %eax
-; SSE-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX-LABEL: test_v16i16_v16i8:
-; AVX: # %bb.0:
-; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX-NEXT: vmovd %xmm0, %eax
-; AVX-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX-NEXT: ret{{[l|q]}}
- %1 = zext <16 x i8> %a0 to <16 x i16>
- %2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %1)
- ret i16 %2
-}
-
-define i16 @test_v32i16_v32i8(<32 x i8> %a0) nounwind {
-; SSE-LABEL: test_v32i16_v32i8:
-; SSE: # %bb.0:
-; SSE-NEXT: pxor %xmm2, %xmm2
-; SSE-NEXT: psadbw %xmm2, %xmm1
-; SSE-NEXT: psadbw %xmm2, %xmm0
-; SSE-NEXT: paddq %xmm1, %xmm0
-; SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE-NEXT: paddq %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE-NEXT: ret{{[l|q]}}
-;
-; AVX1-LABEL: test_v32i16_v32i8:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX1-NEXT: vpsadbw %xmm2, %xmm1, %xmm1
-; AVX1-NEXT: vpsadbw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX1-NEXT: vzeroupper
-; AVX1-NEXT: ret{{[l|q]}}
-;
-; AVX2-LABEL: test_v32i16_v32i8:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: ret{{[l|q]}}
;
@@ -794,8 +1757,8 @@ define i16 @test_v32i16_v32i8(<32 x i8> %a0) nounwind {
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: vmovq %xmm0, %rax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $rax
; AVX512-NEXT: vzeroupper
; AVX512-NEXT: retq
%1 = zext <32 x i8> %a0 to <32 x i16>
@@ -804,73 +1767,222 @@ define i16 @test_v32i16_v32i8(<32 x i8> %a0) nounwind {
}
define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
-; X86-SSE-LABEL: test_v64i16_v64i8:
-; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: pushl %ebp
-; X86-SSE-NEXT: movl %esp, %ebp
-; X86-SSE-NEXT: andl $-16, %esp
-; X86-SSE-NEXT: subl $16, %esp
-; X86-SSE-NEXT: pxor %xmm3, %xmm3
-; X86-SSE-NEXT: psadbw %xmm3, %xmm2
-; X86-SSE-NEXT: psadbw %xmm3, %xmm0
-; X86-SSE-NEXT: paddq %xmm2, %xmm0
-; X86-SSE-NEXT: psadbw %xmm3, %xmm1
-; X86-SSE-NEXT: psadbw 8(%ebp), %xmm3
-; X86-SSE-NEXT: paddq %xmm1, %xmm3
-; X86-SSE-NEXT: paddq %xmm0, %xmm3
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm3[2,3,2,3]
-; X86-SSE-NEXT: paddq %xmm3, %xmm0
-; X86-SSE-NEXT: movd %xmm0, %eax
-; X86-SSE-NEXT: # kill: def $ax killed $ax killed $eax
-; X86-SSE-NEXT: movl %ebp, %esp
-; X86-SSE-NEXT: popl %ebp
-; X86-SSE-NEXT: retl
-;
-; X64-SSE-LABEL: test_v64i16_v64i8:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pxor %xmm4, %xmm4
-; X64-SSE-NEXT: psadbw %xmm4, %xmm3
-; X64-SSE-NEXT: psadbw %xmm4, %xmm1
-; X64-SSE-NEXT: paddq %xmm3, %xmm1
-; X64-SSE-NEXT: psadbw %xmm4, %xmm2
-; X64-SSE-NEXT: psadbw %xmm4, %xmm0
-; X64-SSE-NEXT: paddq %xmm2, %xmm0
-; X64-SSE-NEXT: paddq %xmm1, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE-NEXT: paddq %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
-; X64-SSE-NEXT: # kill: def $ax killed $ax killed $eax
-; X64-SSE-NEXT: retq
-;
-; AVX1-LABEL: test_v64i16_v64i8:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
-; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX1-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
-; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
-; AVX1-NEXT: vpsadbw %xmm3, %xmm4, %xmm4
-; AVX1-NEXT: vpaddq %xmm2, %xmm4, %xmm2
-; AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
-; AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX1-NEXT: vzeroupper
-; AVX1-NEXT: ret{{[l|q]}}
+; X86-SSE2-LABEL: test_v64i16_v64i8:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pushl %ebp
+; X86-SSE2-NEXT: movl %esp, %ebp
+; X86-SSE2-NEXT: andl $-16, %esp
+; X86-SSE2-NEXT: subl $16, %esp
+; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT: pxor %xmm4, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm5
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm6
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; X86-SSE2-NEXT: paddw %xmm5, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm7
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3],xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7]
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm5
+; X86-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X86-SSE2-NEXT: paddw %xmm7, %xmm5
+; X86-SSE2-NEXT: paddw %xmm6, %xmm5
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
+; X86-SSE2-NEXT: paddw %xmm2, %xmm0
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
+; X86-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; X86-SSE2-NEXT: paddw %xmm3, %xmm1
+; X86-SSE2-NEXT: paddw %xmm0, %xmm1
+; X86-SSE2-NEXT: paddw %xmm5, %xmm1
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE2-NEXT: paddw %xmm1, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT: paddw %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: psrld $16, %xmm0
+; X86-SSE2-NEXT: paddw %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE2-NEXT: movl %ebp, %esp
+; X86-SSE2-NEXT: popl %ebp
+; X86-SSE2-NEXT: retl
+;
+; X64-SSE2-LABEL: test_v64i16_v64i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pxor %xmm4, %xmm4
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm5
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm6
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3],xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
+; X64-SSE2-NEXT: paddw %xmm5, %xmm6
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm5
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm7
+; X64-SSE2-NEXT: punpcklbw {{.*#+}} xmm7 = xmm7[0],xmm4[0],xmm7[1],xmm4[1],xmm7[2],xmm4[2],xmm7[3],xmm4[3],xmm7[4],xmm4[4],xmm7[5],xmm4[5],xmm7[6],xmm4[6],xmm7[7],xmm4[7]
+; X64-SSE2-NEXT: paddw %xmm5, %xmm7
+; X64-SSE2-NEXT: paddw %xmm6, %xmm7
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
+; X64-SSE2-NEXT: paddw %xmm2, %xmm0
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
+; X64-SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; X64-SSE2-NEXT: paddw %xmm3, %xmm1
+; X64-SSE2-NEXT: paddw %xmm0, %xmm1
+; X64-SSE2-NEXT: paddw %xmm7, %xmm1
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE2-NEXT: paddw %xmm1, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE2-NEXT: paddw %xmm0, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE2-NEXT: psrld $16, %xmm0
+; X64-SSE2-NEXT: paddw %xmm1, %xmm0
+; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: test_v64i16_v64i8:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: pushl %ebp
+; X86-SSE4-NEXT: movl %esp, %ebp
+; X86-SSE4-NEXT: andl $-16, %esp
+; X86-SSE4-NEXT: subl $16, %esp
+; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm5
+; X86-SSE4-NEXT: pxor %xmm6, %xmm6
+; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm4 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; X86-SSE4-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm6[8],xmm2[9],xmm6[9],xmm2[10],xmm6[10],xmm2[11],xmm6[11],xmm2[12],xmm6[12],xmm2[13],xmm6[13],xmm2[14],xmm6[14],xmm2[15],xmm6[15]
+; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X86-SSE4-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm6[8],xmm0[9],xmm6[9],xmm0[10],xmm6[10],xmm0[11],xmm6[11],xmm0[12],xmm6[12],xmm0[13],xmm6[13],xmm0[14],xmm6[14],xmm0[15],xmm6[15]
+; X86-SSE4-NEXT: paddw %xmm2, %xmm0
+; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm7 = xmm5[0],zero,xmm5[1],zero,xmm5[2],zero,xmm5[3],zero,xmm5[4],zero,xmm5[5],zero,xmm5[6],zero,xmm5[7],zero
+; X86-SSE4-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm6[8],xmm5[9],xmm6[9],xmm5[10],xmm6[10],xmm5[11],xmm6[11],xmm5[12],xmm6[12],xmm5[13],xmm6[13],xmm5[14],xmm6[14],xmm5[15],xmm6[15]
+; X86-SSE4-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; X86-SSE4-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm6[8],xmm1[9],xmm6[9],xmm1[10],xmm6[10],xmm1[11],xmm6[11],xmm1[12],xmm6[12],xmm1[13],xmm6[13],xmm1[14],xmm6[14],xmm1[15],xmm6[15]
+; X86-SSE4-NEXT: paddw %xmm5, %xmm1
+; X86-SSE4-NEXT: paddw %xmm0, %xmm1
+; X86-SSE4-NEXT: paddw %xmm4, %xmm3
+; X86-SSE4-NEXT: paddw %xmm7, %xmm2
+; X86-SSE4-NEXT: paddw %xmm3, %xmm2
+; X86-SSE4-NEXT: paddw %xmm1, %xmm2
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; X86-SSE4-NEXT: paddw %xmm2, %xmm0
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: paddw %xmm0, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE4-NEXT: psrld $16, %xmm0
+; X86-SSE4-NEXT: paddw %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE4-NEXT: movl %ebp, %esp
+; X86-SSE4-NEXT: popl %ebp
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: test_v64i16_v64i8:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pxor %xmm4, %xmm4
+; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm5 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero,xmm2[4],zero,xmm2[5],zero,xmm2[6],zero,xmm2[7],zero
+; X64-SSE4-NEXT: punpckhbw {{.*#+}} xmm2 = xmm2[8],xmm4[8],xmm2[9],xmm4[9],xmm2[10],xmm4[10],xmm2[11],xmm4[11],xmm2[12],xmm4[12],xmm2[13],xmm4[13],xmm2[14],xmm4[14],xmm2[15],xmm4[15]
+; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm6 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; X64-SSE4-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm4[8],xmm0[9],xmm4[9],xmm0[10],xmm4[10],xmm0[11],xmm4[11],xmm0[12],xmm4[12],xmm0[13],xmm4[13],xmm0[14],xmm4[14],xmm0[15],xmm4[15]
+; X64-SSE4-NEXT: paddw %xmm2, %xmm0
+; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm2 = xmm3[0],zero,xmm3[1],zero,xmm3[2],zero,xmm3[3],zero,xmm3[4],zero,xmm3[5],zero,xmm3[6],zero,xmm3[7],zero
+; X64-SSE4-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm4[8],xmm3[9],xmm4[9],xmm3[10],xmm4[10],xmm3[11],xmm4[11],xmm3[12],xmm4[12],xmm3[13],xmm4[13],xmm3[14],xmm4[14],xmm3[15],xmm4[15]
+; X64-SSE4-NEXT: pmovzxbw {{.*#+}} xmm7 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; X64-SSE4-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm4[8],xmm1[9],xmm4[9],xmm1[10],xmm4[10],xmm1[11],xmm4[11],xmm1[12],xmm4[12],xmm1[13],xmm4[13],xmm1[14],xmm4[14],xmm1[15],xmm4[15]
+; X64-SSE4-NEXT: paddw %xmm3, %xmm1
+; X64-SSE4-NEXT: paddw %xmm0, %xmm1
+; X64-SSE4-NEXT: paddw %xmm5, %xmm6
+; X64-SSE4-NEXT: paddw %xmm2, %xmm7
+; X64-SSE4-NEXT: paddw %xmm6, %xmm7
+; X64-SSE4-NEXT: paddw %xmm1, %xmm7
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm7[2,3,2,3]
+; X64-SSE4-NEXT: paddw %xmm7, %xmm0
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: paddw %xmm0, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm0
+; X64-SSE4-NEXT: psrld $16, %xmm0
+; X64-SSE4-NEXT: paddw %xmm1, %xmm0
+; X64-SSE4-NEXT: movd %xmm0, %eax
+; X64-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE4-NEXT: retq
+;
+; AVX1-SLOW-LABEL: test_v64i16_v64i8:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; AVX1-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm4, %xmm3
+; AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm4
+; AVX1-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
+; AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm6
+; AVX1-SLOW-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15]
+; AVX1-SLOW-NEXT: vpaddw %xmm5, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm3, %xmm2
+; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX1-SLOW-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm3, %xmm1
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vmovd %xmm0, %eax
+; AVX1-SLOW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-SLOW-NEXT: vzeroupper
+; AVX1-SLOW-NEXT: ret{{[l|q]}}
+;
+; AVX1-FAST-LABEL: test_v64i16_v64i8:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX1-FAST-NEXT: vpunpckhbw {{.*#+}} xmm3 = xmm1[8],xmm2[8],xmm1[9],xmm2[9],xmm1[10],xmm2[10],xmm1[11],xmm2[11],xmm1[12],xmm2[12],xmm1[13],xmm2[13],xmm1[14],xmm2[14],xmm1[15],xmm2[15]
+; AVX1-FAST-NEXT: vpunpckhbw {{.*#+}} xmm4 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15]
+; AVX1-FAST-NEXT: vpaddw %xmm3, %xmm4, %xmm3
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm4
+; AVX1-FAST-NEXT: vpunpckhbw {{.*#+}} xmm5 = xmm4[8],xmm2[8],xmm4[9],xmm2[9],xmm4[10],xmm2[10],xmm4[11],xmm2[11],xmm4[12],xmm2[12],xmm4[13],xmm2[13],xmm4[14],xmm2[14],xmm4[15],xmm2[15]
+; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm6
+; AVX1-FAST-NEXT: vpunpckhbw {{.*#+}} xmm2 = xmm6[8],xmm2[8],xmm6[9],xmm2[9],xmm6[10],xmm2[10],xmm6[11],xmm2[11],xmm6[12],xmm2[12],xmm6[13],xmm2[13],xmm6[14],xmm2[14],xmm6[15],xmm2[15]
+; AVX1-FAST-NEXT: vpaddw %xmm5, %xmm2, %xmm2
+; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm3, %xmm2
+; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero
+; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm4[0],zero,xmm4[1],zero,xmm4[2],zero,xmm4[3],zero,xmm4[4],zero,xmm4[5],zero,xmm4[6],zero,xmm4[7],zero
+; AVX1-FAST-NEXT: vpmovzxbw {{.*#+}} xmm3 = xmm6[0],zero,xmm6[1],zero,xmm6[2],zero,xmm6[3],zero,xmm6[4],zero,xmm6[5],zero,xmm6[6],zero,xmm6[7],zero
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm3, %xmm1
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vmovd %xmm0, %eax
+; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-FAST-NEXT: vzeroupper
+; AVX1-FAST-NEXT: ret{{[l|q]}}
;
; AVX2-LABEL: test_v64i16_v64i8:
; AVX2: # %bb.0:
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpsadbw %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpsadbw %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm2 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm3 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT: vpaddw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vextracti128 $1, %ymm1, %xmm1
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero,xmm1[8],zero,xmm1[9],zero,xmm1[10],zero,xmm1[11],zero,xmm1[12],zero,xmm1[13],zero,xmm1[14],zero,xmm1[15],zero
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm0
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero,xmm0[8],zero,xmm0[9],zero,xmm0[10],zero,xmm0[11],zero,xmm0[12],zero,xmm0[13],zero,xmm0[14],zero,xmm0[15],zero
+; AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm0, %ymm2, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vmovd %xmm0, %eax
; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
; AVX2-NEXT: vzeroupper
@@ -878,14 +1990,20 @@ define i16 @test_v64i16_v64i8(<64 x i8> %a0) nounwind {
;
; AVX512-LABEL: test_v64i16_v64i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpmovzxbw {{.*#+}} zmm1 = ymm1[0],zero,ymm1[1],zero,ymm1[2],zero,ymm1[3],zero,ymm1[4],zero,ymm1[5],zero,ymm1[6],zero,ymm1[7],zero,ymm1[8],zero,ymm1[9],zero,ymm1[10],zero,ymm1[11],zero,ymm1[12],zero,ymm1[13],zero,ymm1[14],zero,ymm1[15],zero,ymm1[16],zero,ymm1[17],zero,ymm1[18],zero,ymm1[19],zero,ymm1[20],zero,ymm1[21],zero,ymm1[22],zero,ymm1[23],zero,ymm1[24],zero,ymm1[25],zero,ymm1[26],zero,ymm1[27],zero,ymm1[28],zero,ymm1[29],zero,ymm1[30],zero,ymm1[31],zero
+; AVX512-NEXT: vpmovzxbw {{.*#+}} zmm0 = ymm0[0],zero,ymm0[1],zero,ymm0[2],zero,ymm0[3],zero,ymm0[4],zero,ymm0[5],zero,ymm0[6],zero,ymm0[7],zero,ymm0[8],zero,ymm0[9],zero,ymm0[10],zero,ymm0[11],zero,ymm0[12],zero,ymm0[13],zero,ymm0[14],zero,ymm0[15],zero,ymm0[16],zero,ymm0[17],zero,ymm0[18],zero,ymm0[19],zero,ymm0[20],zero,ymm0[21],zero,ymm0[22],zero,ymm0[23],zero,ymm0[24],zero,ymm0[25],zero,ymm0[26],zero,ymm0[27],zero,ymm0[28],zero,ymm0[29],zero,ymm0[30],zero,ymm0[31],zero
+; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
; AVX512-NEXT: vzeroupper
@@ -922,10 +2040,15 @@ declare i8 @llvm.vector.reduce.add.v32i8(<32 x i8>)
declare i8 @llvm.vector.reduce.add.v64i8(<64 x i8>)
declare i8 @llvm.vector.reduce.add.v128i8(<128 x i8>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; AVX: {{.*}}
+; AVX1: {{.*}}
; AVX512BW: {{.*}}
; AVX512BWVL: {{.*}}
+; SSE: {{.*}}
; SSE41: {{.*}}
; X64-AVX1-FAST: {{.*}}
; X64-AVX1-SLOW: {{.*}}
+; X64-SSE: {{.*}}
; X86-AVX1-FAST: {{.*}}
; X86-AVX1-SLOW: {{.*}}
+; X86-SSE: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-add.ll b/llvm/test/CodeGen/X86/vector-reduce-add.ll
index ff5de0e587505..0c754d3738a25 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-add.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-add.ll
@@ -265,7 +265,7 @@ define i64 @test_v8i64(<8 x i64> %a0) nounwind {
; AVX512-LABEL: test_v8i64:
; AVX512: # %bb.0:
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -285,17 +285,17 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE2-NEXT: paddq 56(%ebp), %xmm2
-; X86-SSE2-NEXT: paddq 24(%ebp), %xmm0
-; X86-SSE2-NEXT: paddq %xmm2, %xmm0
; X86-SSE2-NEXT: paddq 72(%ebp), %xmm3
; X86-SSE2-NEXT: paddq 40(%ebp), %xmm1
; X86-SSE2-NEXT: paddq %xmm3, %xmm1
-; X86-SSE2-NEXT: paddq %xmm0, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE2-NEXT: paddq 56(%ebp), %xmm2
+; X86-SSE2-NEXT: paddq 24(%ebp), %xmm0
+; X86-SSE2-NEXT: paddq %xmm2, %xmm0
; X86-SSE2-NEXT: paddq %xmm1, %xmm0
-; X86-SSE2-NEXT: movd %xmm0, %eax
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE2-NEXT: paddq %xmm0, %xmm1
+; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; X86-SSE2-NEXT: movd %xmm0, %edx
; X86-SSE2-NEXT: movl %ebp, %esp
; X86-SSE2-NEXT: popl %ebp
@@ -303,16 +303,16 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
;
; X64-SSE-LABEL: test_v16i64:
; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: paddq %xmm6, %xmm2
-; X64-SSE-NEXT: paddq %xmm4, %xmm0
-; X64-SSE-NEXT: paddq %xmm2, %xmm0
; X64-SSE-NEXT: paddq %xmm7, %xmm3
; X64-SSE-NEXT: paddq %xmm5, %xmm1
; X64-SSE-NEXT: paddq %xmm3, %xmm1
-; X64-SSE-NEXT: paddq %xmm0, %xmm1
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddq %xmm6, %xmm2
+; X64-SSE-NEXT: paddq %xmm4, %xmm0
+; X64-SSE-NEXT: paddq %xmm2, %xmm0
; X64-SSE-NEXT: paddq %xmm1, %xmm0
-; X64-SSE-NEXT: movq %xmm0, %rax
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE-NEXT: paddq %xmm0, %xmm1
+; X64-SSE-NEXT: movq %xmm1, %rax
; X64-SSE-NEXT: retq
;
; X86-SSE4-LABEL: test_v16i64:
@@ -322,17 +322,17 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE4-NEXT: paddq 56(%ebp), %xmm2
-; X86-SSE4-NEXT: paddq 24(%ebp), %xmm0
-; X86-SSE4-NEXT: paddq %xmm2, %xmm0
; X86-SSE4-NEXT: paddq 72(%ebp), %xmm3
; X86-SSE4-NEXT: paddq 40(%ebp), %xmm1
; X86-SSE4-NEXT: paddq %xmm3, %xmm1
-; X86-SSE4-NEXT: paddq %xmm0, %xmm1
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: paddq 56(%ebp), %xmm2
+; X86-SSE4-NEXT: paddq 24(%ebp), %xmm0
+; X86-SSE4-NEXT: paddq %xmm2, %xmm0
; X86-SSE4-NEXT: paddq %xmm1, %xmm0
-; X86-SSE4-NEXT: movd %xmm0, %eax
-; X86-SSE4-NEXT: pextrd $1, %xmm0, %edx
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm0, %xmm1
+; X86-SSE4-NEXT: movd %xmm1, %eax
+; X86-SSE4-NEXT: pextrd $1, %xmm1, %edx
; X86-SSE4-NEXT: movl %ebp, %esp
; X86-SSE4-NEXT: popl %ebp
; X86-SSE4-NEXT: retl
@@ -343,16 +343,16 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-AVX1-NEXT: movl %esp, %ebp
; X86-AVX1-NEXT: andl $-32, %esp
; X86-AVX1-NEXT: subl $32, %esp
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm3
-; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-NEXT: vpaddq %xmm3, %xmm4, %xmm3
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-NEXT: vpaddq 24(%ebp), %xmm4, %xmm4
+; X86-AVX1-NEXT: vpaddq %xmm4, %xmm3, %xmm3
; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-NEXT: vpaddq 24(%ebp), %xmm1, %xmm1
+; X86-AVX1-NEXT: vpaddq 8(%ebp), %xmm1, %xmm1
; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm0, %xmm2, %xmm0
+; X86-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
@@ -364,17 +364,17 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
;
; X64-AVX1-LABEL: test_v16i64:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm4
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-NEXT: vpaddq %xmm5, %xmm6, %xmm5
; X64-AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-NEXT: vpaddq %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm0, %xmm4, %xmm0
+; X64-AVX1-NEXT: vpaddq %xmm4, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovq %xmm0, %rax
@@ -418,7 +418,7 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddq %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -545,7 +545,7 @@ define i32 @test_v8i32(<8 x i32> %a0) nounwind {
; AVX1-FAST-LABEL: test_v8i32:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
@@ -632,8 +632,7 @@ define i32 @test_v16i32(<16 x i32> %a0) nounwind {
; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: vzeroupper
@@ -655,7 +654,7 @@ define i32 @test_v16i32(<16 x i32> %a0) nounwind {
; AVX512-LABEL: test_v16i32:
; AVX512: # %bb.0:
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -677,36 +676,36 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X86-SSE-NEXT: andl $-16, %esp
; X86-SSE-NEXT: subl $16, %esp
; X86-SSE-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE-NEXT: paddd 56(%ebp), %xmm2
-; X86-SSE-NEXT: paddd 24(%ebp), %xmm0
-; X86-SSE-NEXT: paddd %xmm2, %xmm0
; X86-SSE-NEXT: paddd 72(%ebp), %xmm3
; X86-SSE-NEXT: paddd 40(%ebp), %xmm1
; X86-SSE-NEXT: paddd %xmm3, %xmm1
-; X86-SSE-NEXT: paddd %xmm0, %xmm1
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE-NEXT: paddd 56(%ebp), %xmm2
+; X86-SSE-NEXT: paddd 24(%ebp), %xmm0
+; X86-SSE-NEXT: paddd %xmm2, %xmm0
; X86-SSE-NEXT: paddd %xmm1, %xmm0
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-SSE-NEXT: paddd %xmm0, %xmm1
-; X86-SSE-NEXT: movd %xmm1, %eax
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X86-SSE-NEXT: paddd %xmm1, %xmm0
+; X86-SSE-NEXT: movd %xmm0, %eax
; X86-SSE-NEXT: movl %ebp, %esp
; X86-SSE-NEXT: popl %ebp
; X86-SSE-NEXT: retl
;
; X64-SSE-LABEL: test_v32i32:
; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: paddd %xmm6, %xmm2
-; X64-SSE-NEXT: paddd %xmm4, %xmm0
-; X64-SSE-NEXT: paddd %xmm2, %xmm0
; X64-SSE-NEXT: paddd %xmm7, %xmm3
; X64-SSE-NEXT: paddd %xmm5, %xmm1
; X64-SSE-NEXT: paddd %xmm3, %xmm1
-; X64-SSE-NEXT: paddd %xmm0, %xmm1
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddd %xmm6, %xmm2
+; X64-SSE-NEXT: paddd %xmm4, %xmm0
+; X64-SSE-NEXT: paddd %xmm2, %xmm0
; X64-SSE-NEXT: paddd %xmm1, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-SSE-NEXT: paddd %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X64-SSE-NEXT: paddd %xmm1, %xmm0
+; X64-SSE-NEXT: movd %xmm0, %eax
; X64-SSE-NEXT: retq
;
; X86-AVX1-SLOW-LABEL: test_v32i32:
@@ -715,16 +714,16 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X86-AVX1-SLOW-NEXT: movl %esp, %ebp
; X86-AVX1-SLOW-NEXT: andl $-32, %esp
; X86-AVX1-SLOW-NEXT: subl $32, %esp
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm3
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddd 24(%ebp), %xmm4, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm3, %xmm3
; X86-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpaddd 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-SLOW-NEXT: vpaddd 24(%ebp), %xmm1, %xmm1
+; X86-AVX1-SLOW-NEXT: vpaddd 8(%ebp), %xmm1, %xmm1
; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm2, %xmm0
+; X86-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm0, %xmm0
; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -737,17 +736,17 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
;
; X64-AVX1-SLOW-LABEL: test_v32i32:
; X64-AVX1-SLOW: # %bb.0:
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm1, %xmm4
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-SLOW-NEXT: vpaddd %xmm5, %xmm6, %xmm5
; X64-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm5, %xmm4
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm1, %xmm1
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm4, %xmm0
+; X64-AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -762,18 +761,17 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X86-AVX1-FAST-NEXT: movl %esp, %ebp
; X86-AVX1-FAST-NEXT: andl $-32, %esp
; X86-AVX1-FAST-NEXT: subl $32, %esp
-; X86-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm3
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm4, %xmm3
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddd 24(%ebp), %xmm4, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm3, %xmm3
; X86-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpaddd 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-FAST-NEXT: vpaddd 24(%ebp), %xmm1, %xmm1
-; X86-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpaddd %xmm0, %xmm2, %xmm0
-; X86-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-FAST-NEXT: vpaddd 8(%ebp), %xmm1, %xmm1
; X86-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X86-AVX1-FAST-NEXT: movl %ebp, %esp
@@ -783,19 +781,18 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
;
; X64-AVX1-FAST-LABEL: test_v32i32:
; X64-AVX1-FAST: # %bb.0:
-; X64-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm4
-; X64-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm5, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-FAST-NEXT: vpaddd %xmm5, %xmm6, %xmm5
; X64-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm5, %xmm4
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm1
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-FAST-NEXT: vpaddd %xmm2, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vpaddd %xmm0, %xmm4, %xmm0
-; X64-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X64-AVX1-FAST-NEXT: vzeroupper
@@ -841,7 +838,7 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -927,8 +924,7 @@ define i16 @test_v4i16(<4 x i16> %a0) nounwind {
;
; AVX1-FAST-LABEL: test_v4i16:
; AVX1-FAST: # %bb.0:
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1052,7 +1048,7 @@ define i16 @test_v16i16(<16 x i16> %a0) nounwind {
; AVX1-FAST-LABEL: test_v16i16:
; AVX1-FAST: # %bb.0:
; AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm1
-; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm1, %xmm0
+; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
@@ -1158,10 +1154,8 @@ define i16 @test_v32i16(<32 x i16> %a0) nounwind {
; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm3, %xmm2
; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; AVX1-FAST-NEXT: vmovd %xmm0, %eax
; AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1187,7 +1181,7 @@ define i16 @test_v32i16(<32 x i16> %a0) nounwind {
; AVX512-LABEL: test_v32i16:
; AVX512: # %bb.0:
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1212,21 +1206,21 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
; X86-SSE-NEXT: andl $-16, %esp
; X86-SSE-NEXT: subl $16, %esp
; X86-SSE-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE-NEXT: paddw 56(%ebp), %xmm2
-; X86-SSE-NEXT: paddw 24(%ebp), %xmm0
-; X86-SSE-NEXT: paddw %xmm2, %xmm0
; X86-SSE-NEXT: paddw 72(%ebp), %xmm3
; X86-SSE-NEXT: paddw 40(%ebp), %xmm1
; X86-SSE-NEXT: paddw %xmm3, %xmm1
-; X86-SSE-NEXT: paddw %xmm0, %xmm1
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE-NEXT: paddw 56(%ebp), %xmm2
+; X86-SSE-NEXT: paddw 24(%ebp), %xmm0
+; X86-SSE-NEXT: paddw %xmm2, %xmm0
; X86-SSE-NEXT: paddw %xmm1, %xmm0
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-SSE-NEXT: paddw %xmm0, %xmm1
-; X86-SSE-NEXT: movdqa %xmm1, %xmm0
-; X86-SSE-NEXT: psrld $16, %xmm0
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; X86-SSE-NEXT: paddw %xmm1, %xmm0
-; X86-SSE-NEXT: movd %xmm0, %eax
+; X86-SSE-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE-NEXT: psrld $16, %xmm1
+; X86-SSE-NEXT: paddw %xmm0, %xmm1
+; X86-SSE-NEXT: movd %xmm1, %eax
; X86-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE-NEXT: movl %ebp, %esp
; X86-SSE-NEXT: popl %ebp
@@ -1234,21 +1228,21 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
;
; X64-SSE-LABEL: test_v64i16:
; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: paddw %xmm6, %xmm2
-; X64-SSE-NEXT: paddw %xmm4, %xmm0
-; X64-SSE-NEXT: paddw %xmm2, %xmm0
; X64-SSE-NEXT: paddw %xmm7, %xmm3
; X64-SSE-NEXT: paddw %xmm5, %xmm1
; X64-SSE-NEXT: paddw %xmm3, %xmm1
-; X64-SSE-NEXT: paddw %xmm0, %xmm1
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: paddw %xmm6, %xmm2
+; X64-SSE-NEXT: paddw %xmm4, %xmm0
+; X64-SSE-NEXT: paddw %xmm2, %xmm0
; X64-SSE-NEXT: paddw %xmm1, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-SSE-NEXT: paddw %xmm0, %xmm1
-; X64-SSE-NEXT: movdqa %xmm1, %xmm0
-; X64-SSE-NEXT: psrld $16, %xmm0
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; X64-SSE-NEXT: paddw %xmm1, %xmm0
-; X64-SSE-NEXT: movd %xmm0, %eax
+; X64-SSE-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE-NEXT: psrld $16, %xmm1
+; X64-SSE-NEXT: paddw %xmm0, %xmm1
+; X64-SSE-NEXT: movd %xmm1, %eax
; X64-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SSE-NEXT: retq
;
@@ -1258,16 +1252,16 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
; X86-AVX1-SLOW-NEXT: movl %esp, %ebp
; X86-AVX1-SLOW-NEXT: andl $-32, %esp
; X86-AVX1-SLOW-NEXT: subl $32, %esp
-; X86-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm3
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm4, %xmm3
+; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddw 24(%ebp), %xmm4, %xmm4
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm3, %xmm3
; X86-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpaddw 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm3, %xmm2
-; X86-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-SLOW-NEXT: vpaddw 24(%ebp), %xmm1, %xmm1
+; X86-AVX1-SLOW-NEXT: vpaddw 8(%ebp), %xmm1, %xmm1
; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X86-AVX1-SLOW-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; X86-AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm0, %xmm0
; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; X86-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1283,17 +1277,17 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
;
; X64-AVX1-SLOW-LABEL: test_v64i16:
; X64-AVX1-SLOW: # %bb.0:
-; X64-AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm1, %xmm4
-; X64-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm5, %xmm6, %xmm5
; X64-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm5, %xmm4
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-SLOW-NEXT: vpaddw %xmm3, %xmm1, %xmm1
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-SLOW-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpaddw %xmm2, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X64-AVX1-SLOW-NEXT: vpaddw %xmm0, %xmm4, %xmm0
+; X64-AVX1-SLOW-NEXT: vpaddw %xmm4, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-SLOW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; X64-AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -1311,20 +1305,18 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
; X86-AVX1-FAST-NEXT: movl %esp, %ebp
; X86-AVX1-FAST-NEXT: andl $-32, %esp
; X86-AVX1-FAST-NEXT: subl $32, %esp
-; X86-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm3
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddw %xmm3, %xmm4, %xmm3
+; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddw 24(%ebp), %xmm4, %xmm4
+; X86-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm3, %xmm3
; X86-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpaddw 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm3, %xmm2
-; X86-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-FAST-NEXT: vpaddw 24(%ebp), %xmm1, %xmm1
-; X86-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpaddw %xmm0, %xmm2, %xmm0
-; X86-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X86-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-AVX1-FAST-NEXT: vpaddw 8(%ebp), %xmm1, %xmm1
; X86-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vpaddw %xmm3, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X86-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X86-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1335,21 +1327,19 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
;
; X64-AVX1-FAST-LABEL: test_v64i16:
; X64-AVX1-FAST: # %bb.0:
-; X64-AVX1-FAST-NEXT: vpaddw %xmm3, %xmm1, %xmm4
-; X64-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm5, %xmm4
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-FAST-NEXT: vpaddw %xmm5, %xmm6, %xmm5
; X64-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm5, %xmm4
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-FAST-NEXT: vpaddw %xmm3, %xmm1, %xmm1
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-FAST-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-FAST-NEXT: vpaddw %xmm2, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vpaddw %xmm0, %xmm4, %xmm0
-; X64-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; X64-AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX1-FAST-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vpaddw %xmm4, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
+; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vphaddw %xmm0, %xmm0, %xmm0
; X64-AVX1-FAST-NEXT: vmovd %xmm0, %eax
; X64-AVX1-FAST-NEXT: # kill: def $ax killed $ax killed $eax
@@ -1402,7 +1392,7 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1424,27 +1414,47 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
;
define i8 @test_v2i8(<2 x i8> %a0) nounwind {
-; SSE-LABEL: test_v2i8:
-; SSE: # %bb.0:
-; SSE-NEXT: movdqa %xmm0, %xmm1
-; SSE-NEXT: psrlw $8, %xmm1
-; SSE-NEXT: paddb %xmm0, %xmm1
-; SSE-NEXT: movd %xmm1, %eax
-; SSE-NEXT: # kill: def $al killed $al killed $eax
-; SSE-NEXT: ret{{[l|q]}}
+; X86-SSE2-LABEL: test_v2i8:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: # kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT: retl
+;
+; X64-SSE2-LABEL: test_v2i8:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE2-NEXT: movd %xmm1, %eax
+; X64-SSE2-NEXT: # kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT: retq
+;
+; SSE4-LABEL: test_v2i8:
+; SSE4: # %bb.0:
+; SSE4-NEXT: pxor %xmm1, %xmm1
+; SSE4-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; SSE4-NEXT: psadbw %xmm1, %xmm0
+; SSE4-NEXT: movd %xmm0, %eax
+; SSE4-NEXT: # kill: def $al killed $al killed $eax
+; SSE4-NEXT: ret{{[l|q]}}
;
; AVX-LABEL: test_v2i8:
; AVX: # %bb.0:
-; AVX-NEXT: vpsrlw $8, %xmm0, %xmm1
-; AVX-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX-NEXT: vmovd %xmm0, %eax
; AVX-NEXT: # kill: def $al killed $al killed $eax
; AVX-NEXT: ret{{[l|q]}}
;
; AVX512-LABEL: test_v2i8:
; AVX512: # %bb.0:
-; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: # kill: def $al killed $al killed $eax
; AVX512-NEXT: retq
@@ -1453,36 +1463,60 @@ define i8 @test_v2i8(<2 x i8> %a0) nounwind {
}
define i8 @test_v2i8_load(ptr %p) nounwind {
-; X86-SSE-LABEL: test_v2i8_load:
-; X86-SSE: # %bb.0:
-; X86-SSE-NEXT: movl {{[0-9]+}}(%esp), %eax
-; X86-SSE-NEXT: movzwl (%eax), %eax
-; X86-SSE-NEXT: movd %eax, %xmm0
-; X86-SSE-NEXT: movdqa %xmm0, %xmm1
-; X86-SSE-NEXT: psrlw $8, %xmm1
-; X86-SSE-NEXT: paddb %xmm0, %xmm1
-; X86-SSE-NEXT: movd %xmm1, %eax
-; X86-SSE-NEXT: # kill: def $al killed $al killed $eax
-; X86-SSE-NEXT: retl
+; X86-SSE2-LABEL: test_v2i8_load:
+; X86-SSE2: # %bb.0:
+; X86-SSE2-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE2-NEXT: movzwl (%eax), %eax
+; X86-SSE2-NEXT: movd %eax, %xmm0
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
+; X86-SSE2-NEXT: pxor %xmm1, %xmm1
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: # kill: def $al killed $al killed $eax
+; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v2i8_load:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: movzwl (%rdi), %eax
-; X64-SSE-NEXT: movd %eax, %xmm0
-; X64-SSE-NEXT: movdqa %xmm0, %xmm1
-; X64-SSE-NEXT: psrlw $8, %xmm1
-; X64-SSE-NEXT: paddb %xmm0, %xmm1
-; X64-SSE-NEXT: movd %xmm1, %eax
-; X64-SSE-NEXT: # kill: def $al killed $al killed $eax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v2i8_load:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: movzwl (%rdi), %eax
+; X64-SSE2-NEXT: movd %eax, %xmm0
+; X64-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0
+; X64-SSE2-NEXT: pxor %xmm1, %xmm1
+; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE2-NEXT: movd %xmm1, %eax
+; X64-SSE2-NEXT: # kill: def $al killed $al killed $eax
+; X64-SSE2-NEXT: retq
+;
+; X86-SSE4-LABEL: test_v2i8_load:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: movl {{[0-9]+}}(%esp), %eax
+; X86-SSE4-NEXT: movzwl (%eax), %eax
+; X86-SSE4-NEXT: movd %eax, %xmm0
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X86-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: # kill: def $al killed $al killed $eax
+; X86-SSE4-NEXT: retl
+;
+; X64-SSE4-LABEL: test_v2i8_load:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: movzwl (%rdi), %eax
+; X64-SSE4-NEXT: movd %eax, %xmm0
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: pblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X64-SSE4-NEXT: psadbw %xmm1, %xmm0
+; X64-SSE4-NEXT: movd %xmm0, %eax
+; X64-SSE4-NEXT: # kill: def $al killed $al killed $eax
+; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: test_v2i8_load:
; X86-AVX1: # %bb.0:
; X86-AVX1-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-AVX1-NEXT: movzwl (%eax), %eax
; X86-AVX1-NEXT: vmovd %eax, %xmm0
-; X86-AVX1-NEXT: vpsrlw $8, %xmm0, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: # kill: def $al killed $al killed $eax
; X86-AVX1-NEXT: retl
@@ -1491,8 +1525,9 @@ define i8 @test_v2i8_load(ptr %p) nounwind {
; X64-AVX1: # %bb.0:
; X64-AVX1-NEXT: movzwl (%rdi), %eax
; X64-AVX1-NEXT: vmovd %eax, %xmm0
-; X64-AVX1-NEXT: vpsrlw $8, %xmm0, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vmovd %xmm0, %eax
; X64-AVX1-NEXT: # kill: def $al killed $al killed $eax
; X64-AVX1-NEXT: retq
@@ -1502,8 +1537,9 @@ define i8 @test_v2i8_load(ptr %p) nounwind {
; X86-AVX2-NEXT: movl {{[0-9]+}}(%esp), %eax
; X86-AVX2-NEXT: movzwl (%eax), %eax
; X86-AVX2-NEXT: vmovd %eax, %xmm0
-; X86-AVX2-NEXT: vpsrlw $8, %xmm0, %xmm1
-; X86-AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX2-NEXT: vmovd %xmm0, %eax
; X86-AVX2-NEXT: # kill: def $al killed $al killed $eax
; X86-AVX2-NEXT: retl
@@ -1512,8 +1548,9 @@ define i8 @test_v2i8_load(ptr %p) nounwind {
; X64-AVX2: # %bb.0:
; X64-AVX2-NEXT: movzwl (%rdi), %eax
; X64-AVX2-NEXT: vmovd %eax, %xmm0
-; X64-AVX2-NEXT: vpsrlw $8, %xmm0, %xmm1
-; X64-AVX2-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vmovd %xmm0, %eax
; X64-AVX2-NEXT: # kill: def $al killed $al killed $eax
; X64-AVX2-NEXT: retq
@@ -1522,8 +1559,9 @@ define i8 @test_v2i8_load(ptr %p) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: movzwl (%rdi), %eax
; AVX512-NEXT: vmovd %eax, %xmm0
-; AVX512-NEXT: vpsrlw $8, %xmm0, %xmm1
-; AVX512-NEXT: vpaddb %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7]
+; AVX512-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vmovd %xmm0, %eax
; AVX512-NEXT: # kill: def $al killed $al killed $eax
; AVX512-NEXT: retq
@@ -1536,9 +1574,10 @@ define i8 @test_v4i8(<4 x i8> %a0) nounwind {
; SSE2-LABEL: test_v4i8:
; SSE2: # %bb.0:
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; SSE2-NEXT: psadbw %xmm1, %xmm0
-; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: xorps %xmm2, %xmm2
+; SSE2-NEXT: movss {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3]
+; SSE2-NEXT: psadbw %xmm1, %xmm2
+; SSE2-NEXT: movd %xmm2, %eax
; SSE2-NEXT: # kill: def $al killed $al killed $eax
; SSE2-NEXT: ret{{[l|q]}}
;
@@ -2094,6 +2133,3 @@ declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)
declare i8 @llvm.vector.reduce.add.v32i8(<32 x i8>)
declare i8 @llvm.vector.reduce.add.v64i8(<64 x i8>)
declare i8 @llvm.vector.reduce.add.v128i8(<128 x i8>)
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; X64-SSE2: {{.*}}
-; X64-SSE4: {{.*}}
diff --git a/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll b/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll
index e936f1e4faf21..2c3720d761dec 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-ctpop.ll
@@ -228,10 +228,9 @@ define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) nounwind {
; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
; X86-SSE2-NEXT: packuswb %xmm2, %xmm1
-; X86-SSE2-NEXT: packuswb %xmm2, %xmm2
-; X86-SSE2-NEXT: paddd %xmm1, %xmm2
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
-; X86-SSE2-NEXT: paddd %xmm2, %xmm0
+; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE2-NEXT: paddq %xmm1, %xmm0
; X86-SSE2-NEXT: movd %xmm0, %eax
; X86-SSE2-NEXT: retl
;
@@ -258,11 +257,11 @@ define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) nounwind {
; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
; X64-SSE2-NEXT: packuswb %xmm2, %xmm1
-; X64-SSE2-NEXT: packuswb %xmm2, %xmm2
-; X64-SSE2-NEXT: paddd %xmm1, %xmm2
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[1,1,1,1]
-; X64-SSE2-NEXT: paddd %xmm2, %xmm0
-; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE2-NEXT: paddq %xmm1, %xmm0
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: # kill: def $eax killed $eax killed $rax
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: reduce_ctpop_v4i32:
@@ -283,10 +282,9 @@ define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) nounwind {
; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
; X86-SSE4-NEXT: psadbw %xmm0, %xmm2
; X86-SSE4-NEXT: packuswb %xmm1, %xmm2
-; X86-SSE4-NEXT: packuswb %xmm1, %xmm1
-; X86-SSE4-NEXT: paddd %xmm2, %xmm1
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
-; X86-SSE4-NEXT: paddd %xmm1, %xmm0
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm2
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm2, %xmm0
; X86-SSE4-NEXT: movd %xmm0, %eax
; X86-SSE4-NEXT: retl
;
@@ -308,58 +306,102 @@ define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) nounwind {
; X64-SSE4-NEXT: psadbw %xmm0, %xmm3
; X64-SSE4-NEXT: psadbw %xmm0, %xmm1
; X64-SSE4-NEXT: packuswb %xmm3, %xmm1
-; X64-SSE4-NEXT: packuswb %xmm3, %xmm3
-; X64-SSE4-NEXT: paddd %xmm1, %xmm3
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm3[1,1,1,1]
-; X64-SSE4-NEXT: paddd %xmm3, %xmm0
-; X64-SSE4-NEXT: movd %xmm0, %eax
+; X64-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm1, %xmm0
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: # kill: def $eax killed $eax killed $rax
; X64-SSE4-NEXT: retq
;
-; AVX1-LABEL: reduce_ctpop_v4i32:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX1-NEXT: vpsadbw %xmm1, %xmm2, %xmm2
-; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpackuswb %xmm2, %xmm2, %xmm1
-; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: ret{{[l|q]}}
+; X86-AVX1-LABEL: reduce_ctpop_v4i32:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: retl
;
-; AVX2-LABEL: reduce_ctpop_v4i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
-; AVX2-NEXT: vpsadbw %xmm1, %xmm2, %xmm2
-; AVX2-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: vpackuswb %xmm2, %xmm2, %xmm1
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: ret{{[l|q]}}
+; X64-AVX1-LABEL: reduce_ctpop_v4i32:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; X64-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX1-NEXT: retq
+;
+; X86-AVX2-LABEL: reduce_ctpop_v4i32:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X86-AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X86-AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; X86-AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm2, %xmm2
+; X86-AVX2-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: reduce_ctpop_v4i32:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X64-AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; X64-AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm2, %xmm2
+; X64-AVX2-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX2-NEXT: retq
;
; AVX512VL-LABEL: reduce_ctpop_v4i32:
; AVX512VL: # %bb.0:
@@ -377,18 +419,22 @@ define i32 @reduce_ctpop_v4i32(<4 x i32> %a0) nounwind {
; AVX512VL-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT: vpmovdb %xmm0, %xmm0
; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovd %xmm0, %eax
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovq %xmm0, %rax
+; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: retq
;
; AVX512VPOPCNT-LABEL: reduce_ctpop_v4i32:
; AVX512VPOPCNT: # %bb.0:
; AVX512VPOPCNT-NEXT: vpopcntd %xmm0, %xmm0
-; AVX512VPOPCNT-NEXT: vpmovdb %xmm0, %xmm0
; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax
+; AVX512VPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VPOPCNT-NEXT: retq
%p0 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %a0)
%r0 = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %p0)
@@ -416,10 +462,11 @@ define i16 @reduce_ctpop_v8i16(<8 x i16> %a0) nounwind {
; X86-SSE2-NEXT: psllw $8, %xmm0
; X86-SSE2-NEXT: paddb %xmm1, %xmm0
; X86-SSE2-NEXT: psrlw $8, %xmm0
-; X86-SSE2-NEXT: packuswb %xmm0, %xmm0
; X86-SSE2-NEXT: pxor %xmm1, %xmm1
; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
-; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE2-NEXT: paddq %xmm1, %xmm0
+; X86-SSE2-NEXT: movd %xmm0, %eax
; X86-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE2-NEXT: retl
;
@@ -443,105 +490,178 @@ define i16 @reduce_ctpop_v8i16(<8 x i16> %a0) nounwind {
; X64-SSE2-NEXT: psllw $8, %xmm0
; X64-SSE2-NEXT: paddb %xmm1, %xmm0
; X64-SSE2-NEXT: psrlw $8, %xmm0
-; X64-SSE2-NEXT: packuswb %xmm0, %xmm0
; X64-SSE2-NEXT: pxor %xmm1, %xmm1
; X64-SSE2-NEXT: psadbw %xmm0, %xmm1
-; X64-SSE2-NEXT: movd %xmm1, %eax
-; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE2-NEXT: paddq %xmm1, %xmm0
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: # kill: def $ax killed $ax killed $rax
; X64-SSE2-NEXT: retq
;
-; SSE4-LABEL: reduce_ctpop_v8i16:
-; SSE4: # %bb.0:
-; SSE4-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; SSE4-NEXT: movdqa %xmm0, %xmm2
-; SSE4-NEXT: pand %xmm1, %xmm2
-; SSE4-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; SSE4-NEXT: movdqa %xmm3, %xmm4
-; SSE4-NEXT: pshufb %xmm2, %xmm4
-; SSE4-NEXT: psrlw $4, %xmm0
-; SSE4-NEXT: pand %xmm1, %xmm0
-; SSE4-NEXT: pshufb %xmm0, %xmm3
-; SSE4-NEXT: paddb %xmm4, %xmm3
-; SSE4-NEXT: movdqa %xmm3, %xmm0
-; SSE4-NEXT: psllw $8, %xmm0
-; SSE4-NEXT: paddb %xmm3, %xmm0
-; SSE4-NEXT: psrlw $8, %xmm0
-; SSE4-NEXT: packuswb %xmm0, %xmm0
-; SSE4-NEXT: pxor %xmm1, %xmm1
-; SSE4-NEXT: psadbw %xmm0, %xmm1
-; SSE4-NEXT: movd %xmm1, %eax
-; SSE4-NEXT: # kill: def $ax killed $ax killed $eax
-; SSE4-NEXT: ret{{[l|q]}}
+; X86-SSE4-LABEL: reduce_ctpop_v8i16:
+; X86-SSE4: # %bb.0:
+; X86-SSE4-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X86-SSE4-NEXT: pand %xmm1, %xmm2
+; X86-SSE4-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm4
+; X86-SSE4-NEXT: pshufb %xmm2, %xmm4
+; X86-SSE4-NEXT: psrlw $4, %xmm0
+; X86-SSE4-NEXT: pand %xmm1, %xmm0
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm3
+; X86-SSE4-NEXT: paddb %xmm4, %xmm3
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
+; X86-SSE4-NEXT: psllw $8, %xmm0
+; X86-SSE4-NEXT: paddb %xmm3, %xmm0
+; X86-SSE4-NEXT: psrlw $8, %xmm0
+; X86-SSE4-NEXT: pxor %xmm1, %xmm1
+; X86-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: paddq %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
+; X86-SSE4-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-SSE4-NEXT: retl
;
-; AVX1-LABEL: reduce_ctpop_v8i16:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: vpsllw $8, %xmm0, %xmm1
-; AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
-; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX1-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
-; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX1-NEXT: ret{{[l|q]}}
+; X64-SSE4-LABEL: reduce_ctpop_v8i16:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X64-SSE4-NEXT: pand %xmm1, %xmm2
+; X64-SSE4-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-SSE4-NEXT: movdqa %xmm3, %xmm4
+; X64-SSE4-NEXT: pshufb %xmm2, %xmm4
+; X64-SSE4-NEXT: psrlw $4, %xmm0
+; X64-SSE4-NEXT: pand %xmm1, %xmm0
+; X64-SSE4-NEXT: pshufb %xmm0, %xmm3
+; X64-SSE4-NEXT: paddb %xmm4, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm3, %xmm0
+; X64-SSE4-NEXT: psllw $8, %xmm0
+; X64-SSE4-NEXT: paddb %xmm3, %xmm0
+; X64-SSE4-NEXT: psrlw $8, %xmm0
+; X64-SSE4-NEXT: pxor %xmm1, %xmm1
+; X64-SSE4-NEXT: psadbw %xmm0, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm1, %xmm0
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-SSE4-NEXT: retq
;
-; AVX2-LABEL: reduce_ctpop_v8i16:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: vpsllw $8, %xmm0, %xmm1
-; AVX2-NEXT: vpaddb %xmm0, %xmm1, %xmm0
-; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX2-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX2-NEXT: ret{{[l|q]}}
+; X86-AVX1-LABEL: reduce_ctpop_v8i16:
+; X86-AVX1: # %bb.0:
+; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsllw $8, %xmm0, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
+; X86-AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovd %xmm0, %eax
+; X86-AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX1-NEXT: retl
;
-; AVX512VL-LABEL: reduce_ctpop_v8i16:
-; AVX512VL: # %bb.0:
-; AVX512VL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm2
-; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX512VL-NEXT: vpshufb %xmm2, %xmm3, %xmm2
-; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0
-; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vpshufb %xmm0, %xmm3, %xmm0
-; AVX512VL-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; AVX512VL-NEXT: vpsllw $8, %xmm0, %xmm1
-; AVX512VL-NEXT: vpaddb %xmm0, %xmm1, %xmm0
-; AVX512VL-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX512VL-NEXT: vpackuswb %xmm0, %xmm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovd %xmm0, %eax
-; AVX512VL-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512VL-NEXT: retq
+; X64-AVX1-LABEL: reduce_ctpop_v8i16:
+; X64-AVX1: # %bb.0:
+; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; X64-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsllw $8, %xmm0, %xmm1
+; X64-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
+; X64-AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX1-NEXT: retq
;
-; AVX512VPOPCNT-LABEL: reduce_ctpop_v8i16:
-; AVX512VPOPCNT: # %bb.0:
+; X86-AVX2-LABEL: reduce_ctpop_v8i16:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X86-AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X86-AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; X86-AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpsllw $8, %xmm0, %xmm1
+; X86-AVX2-NEXT: vpaddb %xmm0, %xmm1, %xmm0
+; X86-AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: reduce_ctpop_v8i16:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X64-AVX2-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX2-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X64-AVX2-NEXT: vpsrlw $4, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; X64-AVX2-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpsllw $8, %xmm0, %xmm1
+; X64-AVX2-NEXT: vpaddb %xmm0, %xmm1, %xmm0
+; X64-AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $ax killed $ax killed $rax
+; X64-AVX2-NEXT: retq
+;
+; AVX512VL-LABEL: reduce_ctpop_v8i16:
+; AVX512VL: # %bb.0:
+; AVX512VL-NEXT: vpbroadcastb {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm2
+; AVX512VL-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; AVX512VL-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; AVX512VL-NEXT: vpsrlw $4, %xmm0, %xmm0
+; AVX512VL-NEXT: vpand %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpshufb %xmm0, %xmm3, %xmm0
+; AVX512VL-NEXT: vpaddb %xmm2, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsllw $8, %xmm0, %xmm1
+; AVX512VL-NEXT: vpaddb %xmm0, %xmm1, %xmm0
+; AVX512VL-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovq %xmm0, %rax
+; AVX512VL-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX512VL-NEXT: retq
+;
+; AVX512VPOPCNT-LABEL: reduce_ctpop_v8i16:
+; AVX512VPOPCNT: # %bb.0:
; AVX512VPOPCNT-NEXT: vpmovzxwd {{.*#+}} ymm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero
; AVX512VPOPCNT-NEXT: vpopcntd %ymm0, %ymm0
-; AVX512VPOPCNT-NEXT: vpmovdb %ymm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpmovdw %ymm0, %xmm0
; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax
-; AVX512VPOPCNT-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax
+; AVX512VPOPCNT-NEXT: # kill: def $ax killed $ax killed $rax
; AVX512VPOPCNT-NEXT: vzeroupper
; AVX512VPOPCNT-NEXT: retq
%p0 = tail call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %a0)
@@ -939,9 +1059,10 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; AVX512VL-NEXT: vpaddb %ymm2, %ymm0, %ymm0
; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
-; AVX512VL-NEXT: vpmovqb %ymm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vmovq %xmm0, %rax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
@@ -949,9 +1070,10 @@ define i64 @reduce_ctpop_v4i64(<4 x i64> %a0) nounwind {
; AVX512VPOPCNT-LABEL: reduce_ctpop_v4i64:
; AVX512VPOPCNT: # %bb.0:
; AVX512VPOPCNT-NEXT: vpopcntq %ymm0, %ymm0
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm0, %xmm0
-; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax
; AVX512VPOPCNT-NEXT: vzeroupper
; AVX512VPOPCNT-NEXT: retq
@@ -1095,104 +1217,131 @@ define i32 @reduce_ctpop_v8i32(<8 x i32> %a0) nounwind {
;
; X86-AVX1-LABEL: reduce_ctpop_v8i32:
; X86-AVX1: # %bb.0:
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3
-; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-AVX1-NEXT: vpand %xmm1, %xmm3, %xmm4
-; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm2 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X86-AVX1-NEXT: vpshufb %xmm4, %xmm2, %xmm4
-; X86-AVX1-NEXT: vpsrlw $4, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpand %xmm1, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm3
-; X86-AVX1-NEXT: vpaddb %xmm4, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm3 = xmm3[0],zero,xmm3[1],zero
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpackuswb %xmm5, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm2, %xmm5
+; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm1
+; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm3, %xmm1
+; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm4
+; X86-AVX1-NEXT: vpand %xmm2, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpshufb %xmm4, %xmm3, %xmm4
+; X86-AVX1-NEXT: vpaddb %xmm1, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm4[0],zero,xmm4[1],zero
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpackuswb %xmm5, %xmm4, %xmm4
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm5
+; X86-AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm5
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm2, %xmm0
+; X86-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
; X86-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm2, %xmm2
; X86-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddd %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X86-AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: vzeroupper
; X86-AVX1-NEXT: retl
;
; X64-AVX1-LABEL: reduce_ctpop_v8i32:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
-; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm3
-; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X64-AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
-; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpand %xmm2, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X64-AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; X64-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm1[2],xmm3[2],xmm1[3],xmm3[3]
-; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm5, %xmm5
-; X64-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero
-; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpackuswb %xmm5, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm5
-; X64-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm2
+; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX1-NEXT: vpshufb %xmm2, %xmm3, %xmm2
+; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm4
+; X64-AVX1-NEXT: vpand %xmm1, %xmm4, %xmm4
+; X64-AVX1-NEXT: vpshufb %xmm4, %xmm3, %xmm4
+; X64-AVX1-NEXT: vpaddb %xmm2, %xmm4, %xmm2
+; X64-AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; X64-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm5, %xmm5
+; X64-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpackuswb %xmm5, %xmm2, %xmm2
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X64-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm5
+; X64-AVX1-NEXT: vpshufb %xmm5, %xmm3, %xmm5
; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpand %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; X64-AVX1-NEXT: vpand %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpshufb %xmm0, %xmm3, %xmm0
; X64-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm2 = xmm0[2],xmm3[2],xmm0[3],xmm3[3]
-; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm0[2],xmm4[2],xmm0[3],xmm4[3]
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
; X64-AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero
-; X64-AVX1-NEXT: vpsadbw %xmm3, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpackuswb %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpackuswb %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm2, %xmm1
+; X64-AVX1-NEXT: vpaddq %xmm0, %xmm1, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; X64-AVX1-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vmovd %xmm0, %eax
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: # kill: def $eax killed $eax killed $rax
; X64-AVX1-NEXT: vzeroupper
; X64-AVX1-NEXT: retq
;
-; AVX2-LABEL: reduce_ctpop_v8i32:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm2
-; AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; AVX2-NEXT: # ymm3 = mem[0,1,0,1]
-; AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
-; AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpshufb %ymm0, %ymm3, %ymm0
-; AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
-; AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
-; AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
-; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
-; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: vzeroupper
-; AVX2-NEXT: ret{{[l|q]}}
+; X86-AVX2-LABEL: reduce_ctpop_v8i32:
+; X86-AVX2: # %bb.0:
+; X86-AVX2-NEXT: vpbroadcastb {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-AVX2-NEXT: vpand %ymm1, %ymm0, %ymm2
+; X86-AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X86-AVX2-NEXT: # ymm3 = mem[0,1,0,1]
+; X86-AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm2
+; X86-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm3, %ymm0
+; X86-AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX2-NEXT: vmovd %xmm0, %eax
+; X86-AVX2-NEXT: vzeroupper
+; X86-AVX2-NEXT: retl
+;
+; X64-AVX2-LABEL: reduce_ctpop_v8i32:
+; X64-AVX2: # %bb.0:
+; X64-AVX2-NEXT: vpbroadcastb {{.*#+}} ymm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-AVX2-NEXT: vpand %ymm1, %ymm0, %ymm2
+; X64-AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
+; X64-AVX2-NEXT: # ymm3 = mem[0,1,0,1]
+; X64-AVX2-NEXT: vpshufb %ymm2, %ymm3, %ymm2
+; X64-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpand %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpshufb %ymm0, %ymm3, %ymm0
+; X64-AVX2-NEXT: vpaddb %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X64-AVX2-NEXT: vpunpckhdq {{.*#+}} ymm2 = ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[6],ymm1[6],ymm0[7],ymm1[7]
+; X64-AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
+; X64-AVX2-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
+; X64-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
+; X64-AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: # kill: def $eax killed $eax killed $rax
+; X64-AVX2-NEXT: vzeroupper
+; X64-AVX2-NEXT: retq
;
; AVX512VL-LABEL: reduce_ctpop_v8i32:
; AVX512VL: # %bb.0:
@@ -1211,20 +1360,27 @@ define i32 @reduce_ctpop_v8i32(<8 x i32> %a0) nounwind {
; AVX512VL-NEXT: vpunpckldq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[4],ymm1[4],ymm0[5],ymm1[5]
; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
; AVX512VL-NEXT: vpackuswb %ymm2, %ymm0, %ymm0
-; AVX512VL-NEXT: vpmovdb %ymm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovd %xmm0, %eax
+; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovq %xmm0, %rax
+; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VPOPCNT-LABEL: reduce_ctpop_v8i32:
; AVX512VPOPCNT: # %bb.0:
; AVX512VPOPCNT-NEXT: vpopcntd %ymm0, %ymm0
-; AVX512VPOPCNT-NEXT: vpmovdb %ymm0, %xmm0
; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
-; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VPOPCNT-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax
+; AVX512VPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VPOPCNT-NEXT: vzeroupper
; AVX512VPOPCNT-NEXT: retq
%p0 = tail call <8 x i32> @llvm.ctpop.v8i32(<8 x i32> %a0)
@@ -1635,9 +1791,12 @@ define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) nounwind {
; AVX512VL-NEXT: vpaddb %zmm2, %zmm0, %zmm0
; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512VL-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
-; AVX512VL-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VL-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vmovq %xmm0, %rax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
@@ -1645,9 +1804,12 @@ define i64 @reduce_ctpop_v8i64(<8 x i64> %a0) nounwind {
; AVX512VPOPCNT-LABEL: reduce_ctpop_v8i64:
; AVX512VPOPCNT: # %bb.0:
; AVX512VPOPCNT-NEXT: vpopcntq %zmm0, %zmm0
-; AVX512VPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VPOPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax
; AVX512VPOPCNT-NEXT: vzeroupper
; AVX512VPOPCNT-NEXT: retq
@@ -2113,24 +2275,31 @@ define i32 @reduce_ctpop_v16i32(<16 x i32> %a0) nounwind {
; AVX512VL-NEXT: vpunpckldq {{.*#+}} zmm0 = zmm0[0],zmm1[0],zmm0[1],zmm1[1],zmm0[4],zmm1[4],zmm0[5],zmm1[5],zmm0[8],zmm1[8],zmm0[9],zmm1[9],zmm0[12],zmm1[12],zmm0[13],zmm1[13]
; AVX512VL-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
; AVX512VL-NEXT: vpackuswb %zmm2, %zmm0, %zmm0
-; AVX512VL-NEXT: vpmovdb %zmm0, %xmm0
-; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VL-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512VL-NEXT: vmovd %xmm0, %eax
+; AVX512VL-NEXT: vmovq %xmm0, %rax
+; AVX512VL-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VL-NEXT: vzeroupper
; AVX512VL-NEXT: retq
;
; AVX512VPOPCNT-LABEL: reduce_ctpop_v16i32:
; AVX512VPOPCNT: # %bb.0:
; AVX512VPOPCNT-NEXT: vpopcntd %zmm0, %zmm0
-; AVX512VPOPCNT-NEXT: vpmovdb %zmm0, %xmm0
; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
+; AVX512VPOPCNT-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VPOPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; AVX512VPOPCNT-NEXT: vmovd %xmm0, %eax
+; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax
+; AVX512VPOPCNT-NEXT: # kill: def $eax killed $eax killed $rax
; AVX512VPOPCNT-NEXT: vzeroupper
; AVX512VPOPCNT-NEXT: retq
%p0 = tail call <16 x i32> @llvm.ctpop.v16i32(<16 x i32> %a0)
@@ -2145,8 +2314,8 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: movl %esp, %ebp
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm5
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm6
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm5
+; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm6
; X86-SSE2-NEXT: movdqa %xmm6, %xmm3
; X86-SSE2-NEXT: psrlw $1, %xmm3
; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm3
@@ -2160,100 +2329,99 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: movdqa %xmm6, %xmm7
; X86-SSE2-NEXT: psrlw $4, %xmm7
; X86-SSE2-NEXT: paddb %xmm6, %xmm7
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm4
; X86-SSE2-NEXT: psrlw $1, %xmm4
; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm4
-; X86-SSE2-NEXT: psubb %xmm4, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
+; X86-SSE2-NEXT: psubb %xmm4, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm4
; X86-SSE2-NEXT: pand %xmm3, %xmm4
-; X86-SSE2-NEXT: psrlw $2, %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: paddb %xmm4, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm6
+; X86-SSE2-NEXT: psrlw $2, %xmm0
+; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: paddb %xmm4, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm6
; X86-SSE2-NEXT: psrlw $4, %xmm6
-; X86-SSE2-NEXT: paddb %xmm1, %xmm6
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm1 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE2-NEXT: pand %xmm1, %xmm7
-; X86-SSE2-NEXT: pand %xmm1, %xmm6
+; X86-SSE2-NEXT: paddb %xmm0, %xmm6
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm0, %xmm7
+; X86-SSE2-NEXT: pand %xmm0, %xmm6
; X86-SSE2-NEXT: paddb %xmm7, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
-; X86-SSE2-NEXT: psrlw $1, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm5, %xmm0
+; X86-SSE2-NEXT: psrlw $1, %xmm0
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; X86-SSE2-NEXT: pand %xmm4, %xmm1
-; X86-SSE2-NEXT: psubb %xmm1, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
+; X86-SSE2-NEXT: pand %xmm4, %xmm0
+; X86-SSE2-NEXT: psubb %xmm0, %xmm5
+; X86-SSE2-NEXT: movdqa %xmm5, %xmm0
+; X86-SSE2-NEXT: pand %xmm3, %xmm0
; X86-SSE2-NEXT: psrlw $2, %xmm5
; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: paddb %xmm1, %xmm5
+; X86-SSE2-NEXT: paddb %xmm0, %xmm5
; X86-SSE2-NEXT: movdqa %xmm5, %xmm7
; X86-SSE2-NEXT: psrlw $4, %xmm7
; X86-SSE2-NEXT: paddb %xmm5, %xmm7
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
-; X86-SSE2-NEXT: psrlw $1, %xmm1
-; X86-SSE2-NEXT: pand %xmm4, %xmm1
-; X86-SSE2-NEXT: psubb %xmm1, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
-; X86-SSE2-NEXT: pand %xmm3, %xmm1
-; X86-SSE2-NEXT: psrlw $2, %xmm5
-; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: paddb %xmm1, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm1
-; X86-SSE2-NEXT: psrlw $4, %xmm1
-; X86-SSE2-NEXT: paddb %xmm5, %xmm1
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE2-NEXT: pand %xmm4, %xmm7
-; X86-SSE2-NEXT: pand %xmm4, %xmm1
-; X86-SSE2-NEXT: paddb %xmm7, %xmm1
-; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm5
-; X86-SSE2-NEXT: paddb %xmm6, %xmm1
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm6
-; X86-SSE2-NEXT: psrlw $1, %xmm6
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; X86-SSE2-NEXT: pand %xmm4, %xmm6
-; X86-SSE2-NEXT: psubb %xmm6, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm6
-; X86-SSE2-NEXT: pand %xmm3, %xmm6
-; X86-SSE2-NEXT: psrlw $2, %xmm5
-; X86-SSE2-NEXT: pand %xmm3, %xmm5
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: psrlw $1, %xmm0
+; X86-SSE2-NEXT: pand %xmm4, %xmm0
+; X86-SSE2-NEXT: psubb %xmm0, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE2-NEXT: pand %xmm3, %xmm0
+; X86-SSE2-NEXT: psrlw $2, %xmm2
+; X86-SSE2-NEXT: pand %xmm3, %xmm2
+; X86-SSE2-NEXT: paddb %xmm0, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm5
+; X86-SSE2-NEXT: psrlw $4, %xmm5
+; X86-SSE2-NEXT: paddb %xmm2, %xmm5
+; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm2
+; X86-SSE2-NEXT: movdqa {{.*#+}} xmm0 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE2-NEXT: pand %xmm0, %xmm7
+; X86-SSE2-NEXT: pand %xmm0, %xmm5
+; X86-SSE2-NEXT: paddb %xmm7, %xmm5
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm7
; X86-SSE2-NEXT: paddb %xmm6, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm7
-; X86-SSE2-NEXT: psrlw $4, %xmm7
-; X86-SSE2-NEXT: paddb %xmm5, %xmm7
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
-; X86-SSE2-NEXT: psrlw $1, %xmm5
-; X86-SSE2-NEXT: pand %xmm4, %xmm5
-; X86-SSE2-NEXT: psubb %xmm5, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
-; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: psrlw $2, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm7, %xmm0
+; X86-SSE2-NEXT: psrlw $1, %xmm0
+; X86-SSE2-NEXT: pand %xmm4, %xmm0
+; X86-SSE2-NEXT: psubb %xmm0, %xmm7
+; X86-SSE2-NEXT: movdqa %xmm7, %xmm0
; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: paddb %xmm5, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm6
+; X86-SSE2-NEXT: psrlw $2, %xmm7
+; X86-SSE2-NEXT: pand %xmm3, %xmm7
+; X86-SSE2-NEXT: paddb %xmm0, %xmm7
+; X86-SSE2-NEXT: movdqa %xmm7, %xmm0
+; X86-SSE2-NEXT: psrlw $4, %xmm0
+; X86-SSE2-NEXT: paddb %xmm7, %xmm0
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm6
+; X86-SSE2-NEXT: psrlw $1, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm7
+; X86-SSE2-NEXT: pand %xmm4, %xmm6
+; X86-SSE2-NEXT: psubb %xmm6, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm6
+; X86-SSE2-NEXT: pand %xmm3, %xmm6
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm3, %xmm1
+; X86-SSE2-NEXT: paddb %xmm6, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm6
; X86-SSE2-NEXT: psrlw $4, %xmm6
-; X86-SSE2-NEXT: paddb %xmm0, %xmm6
-; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm0
+; X86-SSE2-NEXT: paddb %xmm1, %xmm6
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm1
; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE2-NEXT: pand %xmm4, %xmm7
+; X86-SSE2-NEXT: pand %xmm4, %xmm0
; X86-SSE2-NEXT: pand %xmm4, %xmm6
-; X86-SSE2-NEXT: paddb %xmm7, %xmm6
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
-; X86-SSE2-NEXT: psrlw $1, %xmm5
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
-; X86-SSE2-NEXT: pand %xmm4, %xmm5
-; X86-SSE2-NEXT: psubb %xmm5, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm5
-; X86-SSE2-NEXT: pand %xmm3, %xmm5
-; X86-SSE2-NEXT: psrlw $2, %xmm0
+; X86-SSE2-NEXT: paddb %xmm0, %xmm6
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE2-NEXT: psrlw $1, %xmm0
+; X86-SSE2-NEXT: pand %xmm7, %xmm0
+; X86-SSE2-NEXT: psubb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm0
; X86-SSE2-NEXT: pand %xmm3, %xmm0
-; X86-SSE2-NEXT: paddb %xmm5, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm7
+; X86-SSE2-NEXT: psrlw $2, %xmm1
+; X86-SSE2-NEXT: pand %xmm3, %xmm1
+; X86-SSE2-NEXT: paddb %xmm0, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm7
; X86-SSE2-NEXT: psrlw $4, %xmm7
-; X86-SSE2-NEXT: paddb %xmm0, %xmm7
+; X86-SSE2-NEXT: paddb %xmm1, %xmm7
; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
; X86-SSE2-NEXT: psrlw $1, %xmm0
-; X86-SSE2-NEXT: pand %xmm4, %xmm0
+; X86-SSE2-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0
; X86-SSE2-NEXT: psubb %xmm0, %xmm2
; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
; X86-SSE2-NEXT: pand %xmm3, %xmm0
@@ -2263,12 +2431,11 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: movdqa %xmm2, %xmm0
; X86-SSE2-NEXT: psrlw $4, %xmm0
; X86-SSE2-NEXT: paddb %xmm2, %xmm0
-; X86-SSE2-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-SSE2-NEXT: pand %xmm2, %xmm7
-; X86-SSE2-NEXT: pand %xmm2, %xmm0
+; X86-SSE2-NEXT: pand %xmm4, %xmm7
+; X86-SSE2-NEXT: pand %xmm4, %xmm0
; X86-SSE2-NEXT: paddb %xmm7, %xmm0
; X86-SSE2-NEXT: paddb %xmm6, %xmm0
-; X86-SSE2-NEXT: paddb %xmm1, %xmm0
+; X86-SSE2-NEXT: paddb %xmm5, %xmm0
; X86-SSE2-NEXT: pxor %xmm1, %xmm1
; X86-SSE2-NEXT: psadbw %xmm0, %xmm1
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
@@ -2282,91 +2449,36 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
;
; X64-SSE2-LABEL: reduce_ctpop_v16i64:
; X64-SSE2: # %bb.0:
-; X64-SSE2-NEXT: movdqa %xmm5, %xmm8
+; X64-SSE2-NEXT: movdqa %xmm4, %xmm8
; X64-SSE2-NEXT: psrlw $1, %xmm8
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm9 = [85,85,85,85,85,85,85,85,85,85,85,85,85,85,85,85]
; X64-SSE2-NEXT: pand %xmm9, %xmm8
-; X64-SSE2-NEXT: psubb %xmm8, %xmm5
+; X64-SSE2-NEXT: psubb %xmm8, %xmm4
; X64-SSE2-NEXT: movdqa {{.*#+}} xmm8 = [51,51,51,51,51,51,51,51,51,51,51,51,51,51,51,51]
-; X64-SSE2-NEXT: movdqa %xmm5, %xmm10
+; X64-SSE2-NEXT: movdqa %xmm4, %xmm10
; X64-SSE2-NEXT: pand %xmm8, %xmm10
-; X64-SSE2-NEXT: psrlw $2, %xmm5
-; X64-SSE2-NEXT: pand %xmm8, %xmm5
-; X64-SSE2-NEXT: paddb %xmm10, %xmm5
-; X64-SSE2-NEXT: movdqa %xmm5, %xmm11
+; X64-SSE2-NEXT: psrlw $2, %xmm4
+; X64-SSE2-NEXT: pand %xmm8, %xmm4
+; X64-SSE2-NEXT: paddb %xmm10, %xmm4
+; X64-SSE2-NEXT: movdqa %xmm4, %xmm11
; X64-SSE2-NEXT: psrlw $4, %xmm11
-; X64-SSE2-NEXT: paddb %xmm5, %xmm11
-; X64-SSE2-NEXT: movdqa {{.*#+}} xmm5 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-SSE2-NEXT: pand %xmm5, %xmm11
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm10
+; X64-SSE2-NEXT: paddb %xmm4, %xmm11
+; X64-SSE2-NEXT: movdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X64-SSE2-NEXT: pand %xmm4, %xmm11
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm10
; X64-SSE2-NEXT: psrlw $1, %xmm10
; X64-SSE2-NEXT: pand %xmm9, %xmm10
-; X64-SSE2-NEXT: psubb %xmm10, %xmm1
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm10
+; X64-SSE2-NEXT: psubb %xmm10, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm10
; X64-SSE2-NEXT: pand %xmm8, %xmm10
-; X64-SSE2-NEXT: psrlw $2, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
-; X64-SSE2-NEXT: paddb %xmm10, %xmm1
-; X64-SSE2-NEXT: movdqa %xmm1, %xmm10
-; X64-SSE2-NEXT: psrlw $4, %xmm10
-; X64-SSE2-NEXT: paddb %xmm1, %xmm10
-; X64-SSE2-NEXT: pand %xmm5, %xmm10
-; X64-SSE2-NEXT: paddb %xmm11, %xmm10
-; X64-SSE2-NEXT: movdqa %xmm7, %xmm1
-; X64-SSE2-NEXT: psrlw $1, %xmm1
-; X64-SSE2-NEXT: pand %xmm9, %xmm1
-; X64-SSE2-NEXT: psubb %xmm1, %xmm7
-; X64-SSE2-NEXT: movdqa %xmm7, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
-; X64-SSE2-NEXT: psrlw $2, %xmm7
-; X64-SSE2-NEXT: pand %xmm8, %xmm7
-; X64-SSE2-NEXT: paddb %xmm1, %xmm7
-; X64-SSE2-NEXT: movdqa %xmm7, %xmm11
-; X64-SSE2-NEXT: psrlw $4, %xmm11
-; X64-SSE2-NEXT: paddb %xmm7, %xmm11
-; X64-SSE2-NEXT: pand %xmm5, %xmm11
-; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
-; X64-SSE2-NEXT: psrlw $1, %xmm1
-; X64-SSE2-NEXT: pand %xmm9, %xmm1
-; X64-SSE2-NEXT: psubb %xmm1, %xmm3
-; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
-; X64-SSE2-NEXT: pand %xmm8, %xmm1
-; X64-SSE2-NEXT: psrlw $2, %xmm3
-; X64-SSE2-NEXT: pand %xmm8, %xmm3
-; X64-SSE2-NEXT: paddb %xmm1, %xmm3
-; X64-SSE2-NEXT: movdqa %xmm3, %xmm1
-; X64-SSE2-NEXT: psrlw $4, %xmm1
-; X64-SSE2-NEXT: paddb %xmm3, %xmm1
-; X64-SSE2-NEXT: pand %xmm5, %xmm1
-; X64-SSE2-NEXT: paddb %xmm11, %xmm1
-; X64-SSE2-NEXT: paddb %xmm10, %xmm1
-; X64-SSE2-NEXT: movdqa %xmm4, %xmm3
-; X64-SSE2-NEXT: psrlw $1, %xmm3
-; X64-SSE2-NEXT: pand %xmm9, %xmm3
-; X64-SSE2-NEXT: psubb %xmm3, %xmm4
-; X64-SSE2-NEXT: movdqa %xmm4, %xmm3
-; X64-SSE2-NEXT: pand %xmm8, %xmm3
-; X64-SSE2-NEXT: psrlw $2, %xmm4
-; X64-SSE2-NEXT: pand %xmm8, %xmm4
-; X64-SSE2-NEXT: paddb %xmm3, %xmm4
-; X64-SSE2-NEXT: movdqa %xmm4, %xmm7
-; X64-SSE2-NEXT: psrlw $4, %xmm7
-; X64-SSE2-NEXT: paddb %xmm4, %xmm7
-; X64-SSE2-NEXT: pand %xmm5, %xmm7
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE2-NEXT: psrlw $1, %xmm3
-; X64-SSE2-NEXT: pand %xmm9, %xmm3
-; X64-SSE2-NEXT: psubb %xmm3, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE2-NEXT: pand %xmm8, %xmm3
; X64-SSE2-NEXT: psrlw $2, %xmm0
; X64-SSE2-NEXT: pand %xmm8, %xmm0
-; X64-SSE2-NEXT: paddb %xmm3, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE2-NEXT: psrlw $4, %xmm3
-; X64-SSE2-NEXT: paddb %xmm0, %xmm3
-; X64-SSE2-NEXT: pand %xmm5, %xmm3
-; X64-SSE2-NEXT: paddb %xmm7, %xmm3
+; X64-SSE2-NEXT: paddb %xmm10, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm10
+; X64-SSE2-NEXT: psrlw $4, %xmm10
+; X64-SSE2-NEXT: paddb %xmm0, %xmm10
+; X64-SSE2-NEXT: pand %xmm4, %xmm10
+; X64-SSE2-NEXT: paddb %xmm11, %xmm10
; X64-SSE2-NEXT: movdqa %xmm6, %xmm0
; X64-SSE2-NEXT: psrlw $1, %xmm0
; X64-SSE2-NEXT: pand %xmm9, %xmm0
@@ -2376,28 +2488,83 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X64-SSE2-NEXT: psrlw $2, %xmm6
; X64-SSE2-NEXT: pand %xmm8, %xmm6
; X64-SSE2-NEXT: paddb %xmm0, %xmm6
-; X64-SSE2-NEXT: movdqa %xmm6, %xmm0
-; X64-SSE2-NEXT: psrlw $4, %xmm0
-; X64-SSE2-NEXT: paddb %xmm6, %xmm0
-; X64-SSE2-NEXT: pand %xmm5, %xmm0
-; X64-SSE2-NEXT: movdqa %xmm2, %xmm4
-; X64-SSE2-NEXT: psrlw $1, %xmm4
-; X64-SSE2-NEXT: pand %xmm9, %xmm4
-; X64-SSE2-NEXT: psubb %xmm4, %xmm2
-; X64-SSE2-NEXT: movdqa %xmm2, %xmm4
-; X64-SSE2-NEXT: pand %xmm8, %xmm4
+; X64-SSE2-NEXT: movdqa %xmm6, %xmm11
+; X64-SSE2-NEXT: psrlw $4, %xmm11
+; X64-SSE2-NEXT: paddb %xmm6, %xmm11
+; X64-SSE2-NEXT: pand %xmm4, %xmm11
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X64-SSE2-NEXT: psrlw $1, %xmm0
+; X64-SSE2-NEXT: pand %xmm9, %xmm0
+; X64-SSE2-NEXT: psubb %xmm0, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X64-SSE2-NEXT: pand %xmm8, %xmm0
; X64-SSE2-NEXT: psrlw $2, %xmm2
; X64-SSE2-NEXT: pand %xmm8, %xmm2
-; X64-SSE2-NEXT: paddb %xmm4, %xmm2
-; X64-SSE2-NEXT: movdqa %xmm2, %xmm4
-; X64-SSE2-NEXT: psrlw $4, %xmm4
-; X64-SSE2-NEXT: paddb %xmm2, %xmm4
-; X64-SSE2-NEXT: pand %xmm5, %xmm4
-; X64-SSE2-NEXT: paddb %xmm0, %xmm4
-; X64-SSE2-NEXT: paddb %xmm3, %xmm4
-; X64-SSE2-NEXT: paddb %xmm1, %xmm4
+; X64-SSE2-NEXT: paddb %xmm0, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm0
+; X64-SSE2-NEXT: psrlw $4, %xmm0
+; X64-SSE2-NEXT: paddb %xmm2, %xmm0
+; X64-SSE2-NEXT: pand %xmm4, %xmm0
+; X64-SSE2-NEXT: paddb %xmm11, %xmm0
+; X64-SSE2-NEXT: paddb %xmm10, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm2
+; X64-SSE2-NEXT: psrlw $1, %xmm2
+; X64-SSE2-NEXT: pand %xmm9, %xmm2
+; X64-SSE2-NEXT: psubb %xmm2, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm2
+; X64-SSE2-NEXT: pand %xmm8, %xmm2
+; X64-SSE2-NEXT: psrlw $2, %xmm5
+; X64-SSE2-NEXT: pand %xmm8, %xmm5
+; X64-SSE2-NEXT: paddb %xmm2, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm6
+; X64-SSE2-NEXT: psrlw $4, %xmm6
+; X64-SSE2-NEXT: paddb %xmm5, %xmm6
+; X64-SSE2-NEXT: pand %xmm4, %xmm6
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X64-SSE2-NEXT: psrlw $1, %xmm2
+; X64-SSE2-NEXT: pand %xmm9, %xmm2
+; X64-SSE2-NEXT: psubb %xmm2, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X64-SSE2-NEXT: pand %xmm8, %xmm2
+; X64-SSE2-NEXT: psrlw $2, %xmm1
+; X64-SSE2-NEXT: pand %xmm8, %xmm1
+; X64-SSE2-NEXT: paddb %xmm2, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm2
+; X64-SSE2-NEXT: psrlw $4, %xmm2
+; X64-SSE2-NEXT: paddb %xmm1, %xmm2
+; X64-SSE2-NEXT: pand %xmm4, %xmm2
+; X64-SSE2-NEXT: paddb %xmm6, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm1
+; X64-SSE2-NEXT: psrlw $1, %xmm1
+; X64-SSE2-NEXT: pand %xmm9, %xmm1
+; X64-SSE2-NEXT: psubb %xmm1, %xmm7
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm1
+; X64-SSE2-NEXT: pand %xmm8, %xmm1
+; X64-SSE2-NEXT: psrlw $2, %xmm7
+; X64-SSE2-NEXT: pand %xmm8, %xmm7
+; X64-SSE2-NEXT: paddb %xmm1, %xmm7
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm1
+; X64-SSE2-NEXT: psrlw $4, %xmm1
+; X64-SSE2-NEXT: paddb %xmm7, %xmm1
+; X64-SSE2-NEXT: pand %xmm4, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm5
+; X64-SSE2-NEXT: psrlw $1, %xmm5
+; X64-SSE2-NEXT: pand %xmm9, %xmm5
+; X64-SSE2-NEXT: psubb %xmm5, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm5
+; X64-SSE2-NEXT: pand %xmm8, %xmm5
+; X64-SSE2-NEXT: psrlw $2, %xmm3
+; X64-SSE2-NEXT: pand %xmm8, %xmm3
+; X64-SSE2-NEXT: paddb %xmm5, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm5
+; X64-SSE2-NEXT: psrlw $4, %xmm5
+; X64-SSE2-NEXT: paddb %xmm3, %xmm5
+; X64-SSE2-NEXT: pand %xmm4, %xmm5
+; X64-SSE2-NEXT: paddb %xmm1, %xmm5
+; X64-SSE2-NEXT: paddb %xmm2, %xmm5
+; X64-SSE2-NEXT: paddb %xmm0, %xmm5
; X64-SSE2-NEXT: pxor %xmm0, %xmm0
-; X64-SSE2-NEXT: psadbw %xmm4, %xmm0
+; X64-SSE2-NEXT: psadbw %xmm5, %xmm0
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-SSE2-NEXT: paddq %xmm0, %xmm1
; X64-SSE2-NEXT: movq %xmm1, %rax
@@ -2408,95 +2575,97 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE4-NEXT: pushl %ebp
; X86-SSE4-NEXT: movl %esp, %ebp
; X86-SSE4-NEXT: andl $-16, %esp
-; X86-SSE4-NEXT: subl $32, %esp
-; X86-SSE4-NEXT: movaps %xmm2, (%esp) # 16-byte Spill
-; X86-SSE4-NEXT: movdqa %xmm0, %xmm2
-; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm5
-; X86-SSE4-NEXT: movdqa {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE4-NEXT: subl $16, %esp
+; X86-SSE4-NEXT: movdqa %xmm2, %xmm4
+; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm5
+; X86-SSE4-NEXT: movdqa {{.*#+}} xmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
; X86-SSE4-NEXT: movdqa %xmm5, %xmm6
-; X86-SSE4-NEXT: pand %xmm4, %xmm6
+; X86-SSE4-NEXT: pand %xmm2, %xmm6
; X86-SSE4-NEXT: movdqa {{.*#+}} xmm3 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
; X86-SSE4-NEXT: pshufb %xmm6, %xmm7
; X86-SSE4-NEXT: psrlw $4, %xmm5
-; X86-SSE4-NEXT: pand %xmm4, %xmm5
+; X86-SSE4-NEXT: pand %xmm2, %xmm5
; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
; X86-SSE4-NEXT: pshufb %xmm5, %xmm6
; X86-SSE4-NEXT: paddb %xmm7, %xmm6
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm5
-; X86-SSE4-NEXT: pand %xmm4, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm5
+; X86-SSE4-NEXT: pand %xmm2, %xmm5
; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
; X86-SSE4-NEXT: pshufb %xmm5, %xmm7
-; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
+; X86-SSE4-NEXT: psrlw $4, %xmm0
+; X86-SSE4-NEXT: pand %xmm2, %xmm0
; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm5
-; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm1
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm5
+; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm0
; X86-SSE4-NEXT: paddb %xmm7, %xmm5
; X86-SSE4-NEXT: paddb %xmm6, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm6
-; X86-SSE4-NEXT: pand %xmm4, %xmm6
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm6
+; X86-SSE4-NEXT: pand %xmm2, %xmm6
; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
; X86-SSE4-NEXT: pshufb %xmm6, %xmm7
-; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
+; X86-SSE4-NEXT: psrlw $4, %xmm0
+; X86-SSE4-NEXT: pand %xmm2, %xmm0
; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm6
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm1
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm6
; X86-SSE4-NEXT: paddb %xmm7, %xmm6
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm7
-; X86-SSE4-NEXT: pand %xmm4, %xmm7
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm0
+; X86-SSE4-NEXT: pand %xmm2, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm7
+; X86-SSE4-NEXT: psrlw $4, %xmm4
+; X86-SSE4-NEXT: pand %xmm2, %xmm4
+; X86-SSE4-NEXT: movdqa %xmm2, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm2
+; X86-SSE4-NEXT: pshufb %xmm4, %xmm2
+; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm4
+; X86-SSE4-NEXT: paddb %xmm7, %xmm2
+; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm7
+; X86-SSE4-NEXT: paddb %xmm6, %xmm2
+; X86-SSE4-NEXT: paddb %xmm5, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm7, %xmm5
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm6
+; X86-SSE4-NEXT: pand %xmm0, %xmm5
; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
-; X86-SSE4-NEXT: pshufb %xmm7, %xmm0
-; X86-SSE4-NEXT: psrlw $4, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
+; X86-SSE4-NEXT: pshufb %xmm5, %xmm0
+; X86-SSE4-NEXT: psrlw $4, %xmm7
+; X86-SSE4-NEXT: movdqa %xmm6, %xmm5
+; X86-SSE4-NEXT: pand %xmm6, %xmm7
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
+; X86-SSE4-NEXT: pshufb %xmm7, %xmm6
+; X86-SSE4-NEXT: paddb %xmm0, %xmm6
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm0
+; X86-SSE4-NEXT: pand %xmm5, %xmm0
; X86-SSE4-NEXT: movdqa %xmm3, %xmm7
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm7
-; X86-SSE4-NEXT: paddb %xmm0, %xmm7
-; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm0
-; X86-SSE4-NEXT: paddb %xmm6, %xmm7
-; X86-SSE4-NEXT: paddb %xmm5, %xmm7
-; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm7
+; X86-SSE4-NEXT: psrlw $4, %xmm1
+; X86-SSE4-NEXT: pand %xmm5, %xmm1
; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
; X86-SSE4-NEXT: pshufb %xmm1, %xmm5
-; X86-SSE4-NEXT: psrlw $4, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm6
-; X86-SSE4-NEXT: paddb %xmm5, %xmm6
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm1
-; X86-SSE4-NEXT: pshufb %xmm0, %xmm1
-; X86-SSE4-NEXT: psrlw $4, %xmm2
-; X86-SSE4-NEXT: pand %xmm4, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm5
-; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm2
-; X86-SSE4-NEXT: paddb %xmm1, %xmm5
+; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm0
+; X86-SSE4-NEXT: paddb %xmm7, %xmm5
; X86-SSE4-NEXT: paddb %xmm6, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm0
-; X86-SSE4-NEXT: pand %xmm4, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE4-NEXT: movdqa {{.*#+}} xmm7 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
+; X86-SSE4-NEXT: pand %xmm7, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
+; X86-SSE4-NEXT: pshufb %xmm1, %xmm6
+; X86-SSE4-NEXT: psrlw $4, %xmm0
+; X86-SSE4-NEXT: pand %xmm7, %xmm0
; X86-SSE4-NEXT: movdqa %xmm3, %xmm1
; X86-SSE4-NEXT: pshufb %xmm0, %xmm1
-; X86-SSE4-NEXT: psrlw $4, %xmm2
-; X86-SSE4-NEXT: pand %xmm4, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm0
-; X86-SSE4-NEXT: pshufb %xmm2, %xmm0
-; X86-SSE4-NEXT: paddb %xmm1, %xmm0
-; X86-SSE4-NEXT: movdqa (%esp), %xmm6 # 16-byte Reload
-; X86-SSE4-NEXT: movdqa %xmm6, %xmm1
-; X86-SSE4-NEXT: pand %xmm4, %xmm1
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm2
-; X86-SSE4-NEXT: pshufb %xmm1, %xmm2
-; X86-SSE4-NEXT: psrlw $4, %xmm6
-; X86-SSE4-NEXT: pand %xmm4, %xmm6
-; X86-SSE4-NEXT: pshufb %xmm6, %xmm3
-; X86-SSE4-NEXT: paddb %xmm2, %xmm3
-; X86-SSE4-NEXT: paddb %xmm0, %xmm3
+; X86-SSE4-NEXT: paddb %xmm6, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm0
+; X86-SSE4-NEXT: pand %xmm7, %xmm0
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
+; X86-SSE4-NEXT: pshufb %xmm0, %xmm6
+; X86-SSE4-NEXT: psrlw $4, %xmm4
+; X86-SSE4-NEXT: pand %xmm7, %xmm4
+; X86-SSE4-NEXT: pshufb %xmm4, %xmm3
+; X86-SSE4-NEXT: paddb %xmm6, %xmm3
+; X86-SSE4-NEXT: paddb %xmm1, %xmm3
; X86-SSE4-NEXT: paddb %xmm5, %xmm3
-; X86-SSE4-NEXT: paddb %xmm7, %xmm3
+; X86-SSE4-NEXT: paddb %xmm2, %xmm3
; X86-SSE4-NEXT: pxor %xmm0, %xmm0
; X86-SSE4-NEXT: psadbw %xmm3, %xmm0
; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -2510,85 +2679,85 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X64-SSE4-LABEL: reduce_ctpop_v16i64:
; X64-SSE4: # %bb.0:
; X64-SSE4-NEXT: movdqa {{.*#+}} xmm9 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-SSE4-NEXT: movdqa %xmm5, %xmm10
+; X64-SSE4-NEXT: movdqa %xmm4, %xmm10
; X64-SSE4-NEXT: pand %xmm9, %xmm10
; X64-SSE4-NEXT: movdqa {{.*#+}} xmm8 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
; X64-SSE4-NEXT: pshufb %xmm10, %xmm11
-; X64-SSE4-NEXT: psrlw $4, %xmm5
-; X64-SSE4-NEXT: pand %xmm9, %xmm5
+; X64-SSE4-NEXT: psrlw $4, %xmm4
+; X64-SSE4-NEXT: pand %xmm9, %xmm4
; X64-SSE4-NEXT: movdqa %xmm8, %xmm10
-; X64-SSE4-NEXT: pshufb %xmm5, %xmm10
+; X64-SSE4-NEXT: pshufb %xmm4, %xmm10
; X64-SSE4-NEXT: paddb %xmm11, %xmm10
-; X64-SSE4-NEXT: movdqa %xmm1, %xmm5
-; X64-SSE4-NEXT: pand %xmm9, %xmm5
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
-; X64-SSE4-NEXT: pshufb %xmm5, %xmm11
-; X64-SSE4-NEXT: psrlw $4, %xmm1
-; X64-SSE4-NEXT: pand %xmm9, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm5
-; X64-SSE4-NEXT: pshufb %xmm1, %xmm5
-; X64-SSE4-NEXT: paddb %xmm11, %xmm5
-; X64-SSE4-NEXT: paddb %xmm10, %xmm5
-; X64-SSE4-NEXT: movdqa %xmm7, %xmm1
-; X64-SSE4-NEXT: pand %xmm9, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm10
-; X64-SSE4-NEXT: pshufb %xmm1, %xmm10
-; X64-SSE4-NEXT: psrlw $4, %xmm7
-; X64-SSE4-NEXT: pand %xmm9, %xmm7
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
-; X64-SSE4-NEXT: pshufb %xmm7, %xmm11
-; X64-SSE4-NEXT: paddb %xmm10, %xmm11
-; X64-SSE4-NEXT: movdqa %xmm3, %xmm1
-; X64-SSE4-NEXT: pand %xmm9, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm7
-; X64-SSE4-NEXT: pshufb %xmm1, %xmm7
-; X64-SSE4-NEXT: psrlw $4, %xmm3
-; X64-SSE4-NEXT: pand %xmm9, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm1
-; X64-SSE4-NEXT: pshufb %xmm3, %xmm1
-; X64-SSE4-NEXT: paddb %xmm7, %xmm1
-; X64-SSE4-NEXT: paddb %xmm11, %xmm1
-; X64-SSE4-NEXT: paddb %xmm5, %xmm1
-; X64-SSE4-NEXT: movdqa %xmm4, %xmm3
-; X64-SSE4-NEXT: pand %xmm9, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm5
-; X64-SSE4-NEXT: pshufb %xmm3, %xmm5
-; X64-SSE4-NEXT: psrlw $4, %xmm4
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm4
; X64-SSE4-NEXT: pand %xmm9, %xmm4
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm7
-; X64-SSE4-NEXT: pshufb %xmm4, %xmm7
-; X64-SSE4-NEXT: paddb %xmm5, %xmm7
-; X64-SSE4-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE4-NEXT: pand %xmm9, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm4
-; X64-SSE4-NEXT: pshufb %xmm3, %xmm4
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
+; X64-SSE4-NEXT: pshufb %xmm4, %xmm11
; X64-SSE4-NEXT: psrlw $4, %xmm0
; X64-SSE4-NEXT: pand %xmm9, %xmm0
-; X64-SSE4-NEXT: movdqa %xmm8, %xmm3
-; X64-SSE4-NEXT: pshufb %xmm0, %xmm3
-; X64-SSE4-NEXT: paddb %xmm4, %xmm3
-; X64-SSE4-NEXT: paddb %xmm7, %xmm3
-; X64-SSE4-NEXT: movdqa %xmm6, %xmm0
-; X64-SSE4-NEXT: pand %xmm9, %xmm0
; X64-SSE4-NEXT: movdqa %xmm8, %xmm4
; X64-SSE4-NEXT: pshufb %xmm0, %xmm4
+; X64-SSE4-NEXT: paddb %xmm11, %xmm4
+; X64-SSE4-NEXT: paddb %xmm10, %xmm4
+; X64-SSE4-NEXT: movdqa %xmm6, %xmm0
+; X64-SSE4-NEXT: pand %xmm9, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm10
+; X64-SSE4-NEXT: pshufb %xmm0, %xmm10
; X64-SSE4-NEXT: psrlw $4, %xmm6
; X64-SSE4-NEXT: pand %xmm9, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm11
+; X64-SSE4-NEXT: pshufb %xmm6, %xmm11
+; X64-SSE4-NEXT: paddb %xmm10, %xmm11
+; X64-SSE4-NEXT: movdqa %xmm2, %xmm0
+; X64-SSE4-NEXT: pand %xmm9, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm6
+; X64-SSE4-NEXT: pshufb %xmm0, %xmm6
+; X64-SSE4-NEXT: psrlw $4, %xmm2
+; X64-SSE4-NEXT: pand %xmm9, %xmm2
; X64-SSE4-NEXT: movdqa %xmm8, %xmm0
-; X64-SSE4-NEXT: pshufb %xmm6, %xmm0
+; X64-SSE4-NEXT: pshufb %xmm2, %xmm0
+; X64-SSE4-NEXT: paddb %xmm6, %xmm0
+; X64-SSE4-NEXT: paddb %xmm11, %xmm0
; X64-SSE4-NEXT: paddb %xmm4, %xmm0
-; X64-SSE4-NEXT: movdqa %xmm2, %xmm4
+; X64-SSE4-NEXT: movdqa %xmm5, %xmm2
+; X64-SSE4-NEXT: pand %xmm9, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm4
+; X64-SSE4-NEXT: pshufb %xmm2, %xmm4
+; X64-SSE4-NEXT: psrlw $4, %xmm5
+; X64-SSE4-NEXT: pand %xmm9, %xmm5
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm6
+; X64-SSE4-NEXT: pshufb %xmm5, %xmm6
+; X64-SSE4-NEXT: paddb %xmm4, %xmm6
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm2
+; X64-SSE4-NEXT: pand %xmm9, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm4
+; X64-SSE4-NEXT: pshufb %xmm2, %xmm4
+; X64-SSE4-NEXT: psrlw $4, %xmm1
+; X64-SSE4-NEXT: pand %xmm9, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm2
+; X64-SSE4-NEXT: pshufb %xmm1, %xmm2
+; X64-SSE4-NEXT: paddb %xmm4, %xmm2
+; X64-SSE4-NEXT: paddb %xmm6, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm7, %xmm1
+; X64-SSE4-NEXT: pand %xmm9, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm4
+; X64-SSE4-NEXT: pshufb %xmm1, %xmm4
+; X64-SSE4-NEXT: psrlw $4, %xmm7
+; X64-SSE4-NEXT: pand %xmm9, %xmm7
+; X64-SSE4-NEXT: movdqa %xmm8, %xmm1
+; X64-SSE4-NEXT: pshufb %xmm7, %xmm1
+; X64-SSE4-NEXT: paddb %xmm4, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm3, %xmm4
; X64-SSE4-NEXT: pand %xmm9, %xmm4
; X64-SSE4-NEXT: movdqa %xmm8, %xmm5
; X64-SSE4-NEXT: pshufb %xmm4, %xmm5
-; X64-SSE4-NEXT: psrlw $4, %xmm2
-; X64-SSE4-NEXT: pand %xmm9, %xmm2
-; X64-SSE4-NEXT: pshufb %xmm2, %xmm8
+; X64-SSE4-NEXT: psrlw $4, %xmm3
+; X64-SSE4-NEXT: pand %xmm9, %xmm3
+; X64-SSE4-NEXT: pshufb %xmm3, %xmm8
; X64-SSE4-NEXT: paddb %xmm5, %xmm8
-; X64-SSE4-NEXT: paddb %xmm0, %xmm8
-; X64-SSE4-NEXT: paddb %xmm3, %xmm8
; X64-SSE4-NEXT: paddb %xmm1, %xmm8
+; X64-SSE4-NEXT: paddb %xmm2, %xmm8
+; X64-SSE4-NEXT: paddb %xmm0, %xmm8
; X64-SSE4-NEXT: pxor %xmm0, %xmm0
; X64-SSE4-NEXT: psadbw %xmm8, %xmm0
; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -2601,72 +2770,70 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X86-AVX1-NEXT: pushl %ebp
; X86-AVX1-NEXT: movl %esp, %ebp
; X86-AVX1-NEXT: andl $-32, %esp
-; X86-AVX1-NEXT: subl $96, %esp
-; X86-AVX1-NEXT: vmovaps %ymm1, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
-; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X86-AVX1-NEXT: subl $32, %esp
; X86-AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm6
+; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm5
; X86-AVX1-NEXT: vmovdqa {{.*#+}} xmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
-; X86-AVX1-NEXT: vpsrlw $4, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm5
; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
-; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm7
-; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
-; X86-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm6
; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm6
; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
-; X86-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
; X86-AVX1-NEXT: vpaddb %xmm5, %xmm6, %xmm5
-; X86-AVX1-NEXT: vmovdqa %xmm5, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-AVX1-NEXT: vmovdqa 24(%ebp), %xmm6
+; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm7
+; X86-AVX1-NEXT: vpand %xmm3, %xmm7, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm7, %xmm6
+; X86-AVX1-NEXT: vpaddb %xmm5, %xmm6, %xmm5
+; X86-AVX1-NEXT: vmovdqa %xmm5, (%esp) # 16-byte Spill
+; X86-AVX1-NEXT: vmovdqa 8(%ebp), %xmm6
; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm7
; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
; X86-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm6
; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
; X86-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm7
-; X86-AVX1-NEXT: vpand %xmm3, %xmm7, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X86-AVX1-NEXT: vpsrlw $4, %xmm7, %xmm7
-; X86-AVX1-NEXT: vpand %xmm3, %xmm7, %xmm7
+; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm7
; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm7, %xmm5
+; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm5
+; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm5, %xmm5
; X86-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpaddb {{[-0-9]+}}(%e{{[sb]}}p), %xmm5, %xmm5 # 16-byte Folded Reload
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpaddb (%esp), %xmm5, %xmm5 # 16-byte Folded Reload
+; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
+; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm2, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm2
-; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm2, %xmm2
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovdqa 24(%ebp), %xmm6
; X86-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vmovdqa 8(%ebp), %xmm2
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm2, %xmm1
-; X86-AVX1-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %ymm6 # 32-byte Reload
; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm2
-; X86-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
-; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm3
; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vpshufb %xmm3, %xmm4, %xmm3
-; X86-AVX1-NEXT: vpaddb %xmm2, %xmm3, %xmm2
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm2, %xmm1
+; X86-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X86-AVX1-NEXT: vpaddb %xmm2, %xmm6, %xmm2
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
+; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm3
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm2, %xmm1, %xmm1
; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm0, %xmm5, %xmm0
; X86-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
; X86-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -2680,45 +2847,43 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
;
; X64-AVX1-LABEL: reduce_ctpop_v16i64:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm6
; X64-AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; X64-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm7
+; X64-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm6
; X64-AVX1-NEXT: vmovdqa {{.*#+}} xmm5 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
-; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
-; X64-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpand %xmm4, %xmm6, %xmm6
; X64-AVX1-NEXT: vpshufb %xmm6, %xmm5, %xmm6
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm7
-; X64-AVX1-NEXT: vpand %xmm4, %xmm7, %xmm8
-; X64-AVX1-NEXT: vpshufb %xmm8, %xmm5, %xmm8
-; X64-AVX1-NEXT: vpsrlw $4, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm7
; X64-AVX1-NEXT: vpand %xmm4, %xmm7, %xmm7
; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
-; X64-AVX1-NEXT: vpaddb %xmm7, %xmm8, %xmm7
; X64-AVX1-NEXT: vpaddb %xmm6, %xmm7, %xmm6
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm7
-; X64-AVX1-NEXT: vpand %xmm4, %xmm7, %xmm8
-; X64-AVX1-NEXT: vpshufb %xmm8, %xmm5, %xmm8
-; X64-AVX1-NEXT: vpsrlw $4, %xmm7, %xmm7
-; X64-AVX1-NEXT: vpand %xmm4, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm7
; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
+; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm8
+; X64-AVX1-NEXT: vpand %xmm4, %xmm8, %xmm8
+; X64-AVX1-NEXT: vpshufb %xmm8, %xmm5, %xmm8
; X64-AVX1-NEXT: vpaddb %xmm7, %xmm8, %xmm7
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm8
-; X64-AVX1-NEXT: vpand %xmm4, %xmm8, %xmm9
-; X64-AVX1-NEXT: vpshufb %xmm9, %xmm5, %xmm9
-; X64-AVX1-NEXT: vpsrlw $4, %xmm8, %xmm8
+; X64-AVX1-NEXT: vpaddb %xmm6, %xmm7, %xmm6
+; X64-AVX1-NEXT: vpand %xmm4, %xmm3, %xmm7
+; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
+; X64-AVX1-NEXT: vpsrlw $4, %xmm3, %xmm8
; X64-AVX1-NEXT: vpand %xmm4, %xmm8, %xmm8
; X64-AVX1-NEXT: vpshufb %xmm8, %xmm5, %xmm8
-; X64-AVX1-NEXT: vpaddb %xmm9, %xmm8, %xmm8
+; X64-AVX1-NEXT: vpaddb %xmm7, %xmm8, %xmm7
+; X64-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm8
+; X64-AVX1-NEXT: vpshufb %xmm8, %xmm5, %xmm8
+; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm9
+; X64-AVX1-NEXT: vpand %xmm4, %xmm9, %xmm9
+; X64-AVX1-NEXT: vpshufb %xmm9, %xmm5, %xmm9
+; X64-AVX1-NEXT: vpaddb %xmm8, %xmm9, %xmm8
; X64-AVX1-NEXT: vpaddb %xmm7, %xmm8, %xmm7
; X64-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
+; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
; X64-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm7
; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
; X64-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
; X64-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2
; X64-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
; X64-AVX1-NEXT: vpaddb %xmm7, %xmm2, %xmm2
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-NEXT: vpand %xmm4, %xmm0, %xmm7
; X64-AVX1-NEXT: vpshufb %xmm7, %xmm5, %xmm7
; X64-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
@@ -2726,12 +2891,14 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X64-AVX1-NEXT: vpshufb %xmm0, %xmm5, %xmm0
; X64-AVX1-NEXT: vpaddb %xmm7, %xmm0, %xmm0
; X64-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpand %xmm4, %xmm3, %xmm2
-; X64-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
-; X64-AVX1-NEXT: vpsrlw $4, %xmm3, %xmm3
-; X64-AVX1-NEXT: vpand %xmm4, %xmm3, %xmm3
+; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm2
+; X64-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm3
; X64-AVX1-NEXT: vpshufb %xmm3, %xmm5, %xmm3
-; X64-AVX1-NEXT: vpaddb %xmm2, %xmm3, %xmm2
+; X64-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpand %xmm4, %xmm2, %xmm2
+; X64-AVX1-NEXT: vpshufb %xmm2, %xmm5, %xmm2
+; X64-AVX1-NEXT: vpaddb %xmm3, %xmm2, %xmm2
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-NEXT: vpand %xmm4, %xmm1, %xmm3
; X64-AVX1-NEXT: vpshufb %xmm3, %xmm5, %xmm3
; X64-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
@@ -2740,7 +2907,7 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; X64-AVX1-NEXT: vpaddb %xmm3, %xmm1, %xmm1
; X64-AVX1-NEXT: vpaddb %xmm2, %xmm1, %xmm1
; X64-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpaddb %xmm0, %xmm6, %xmm0
; X64-AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
; X64-AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -2844,28 +3011,27 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
; AVX512VL-LABEL: reduce_ctpop_v16i64:
; AVX512VL: # %bb.0:
; AVX512VL-NEXT: vpbroadcastb {{.*#+}} zmm2 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
-; AVX512VL-NEXT: vpandq %zmm2, %zmm0, %zmm3
+; AVX512VL-NEXT: vpandq %zmm2, %zmm1, %zmm3
; AVX512VL-NEXT: vbroadcasti32x4 {{.*#+}} zmm4 = [0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4,0,1,1,2,1,2,2,3,1,2,2,3,2,3,3,4]
; AVX512VL-NEXT: # zmm4 = mem[0,1,2,3,0,1,2,3,0,1,2,3,0,1,2,3]
; AVX512VL-NEXT: vpshufb %zmm3, %zmm4, %zmm3
+; AVX512VL-NEXT: vpsrlw $4, %zmm1, %zmm1
+; AVX512VL-NEXT: vpandq %zmm2, %zmm1, %zmm1
+; AVX512VL-NEXT: vpshufb %zmm1, %zmm4, %zmm1
+; AVX512VL-NEXT: vpaddb %zmm3, %zmm1, %zmm1
+; AVX512VL-NEXT: vpandq %zmm2, %zmm0, %zmm3
+; AVX512VL-NEXT: vpshufb %zmm3, %zmm4, %zmm3
; AVX512VL-NEXT: vpsrlw $4, %zmm0, %zmm0
; AVX512VL-NEXT: vpandq %zmm2, %zmm0, %zmm0
; AVX512VL-NEXT: vpshufb %zmm0, %zmm4, %zmm0
; AVX512VL-NEXT: vpaddb %zmm3, %zmm0, %zmm0
-; AVX512VL-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX512VL-NEXT: vpsadbw %zmm3, %zmm0, %zmm0
-; AVX512VL-NEXT: vpandq %zmm2, %zmm1, %zmm5
-; AVX512VL-NEXT: vpshufb %zmm5, %zmm4, %zmm5
-; AVX512VL-NEXT: vpsrlw $4, %zmm1, %zmm1
-; AVX512VL-NEXT: vpandq %zmm2, %zmm1, %zmm1
-; AVX512VL-NEXT: vpshufb %zmm1, %zmm4, %zmm1
-; AVX512VL-NEXT: vpaddb %zmm5, %zmm1, %zmm1
-; AVX512VL-NEXT: vpsadbw %zmm3, %zmm1, %zmm1
-; AVX512VL-NEXT: vpmovqb %zmm1, %xmm1
-; AVX512VL-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512VL-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX512VL-NEXT: vpaddb %zmm1, %zmm0, %zmm0
; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VL-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpsadbw %zmm1, %zmm0, %zmm0
+; AVX512VL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VL-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VL-NEXT: vmovq %xmm0, %rax
@@ -2874,13 +3040,13 @@ define i64 @reduce_ctpop_v16i64(<16 x i64> %a0) nounwind {
;
; AVX512VPOPCNT-LABEL: reduce_ctpop_v16i64:
; AVX512VPOPCNT: # %bb.0:
-; AVX512VPOPCNT-NEXT: vpopcntq %zmm0, %zmm0
; AVX512VPOPCNT-NEXT: vpopcntq %zmm1, %zmm1
-; AVX512VPOPCNT-NEXT: vpmovqb %zmm1, %xmm1
-; AVX512VPOPCNT-NEXT: vpmovqb %zmm0, %xmm0
-; AVX512VPOPCNT-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; AVX512VPOPCNT-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512VPOPCNT-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpopcntq %zmm0, %zmm0
+; AVX512VPOPCNT-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512VPOPCNT-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512VPOPCNT-NEXT: vpaddq %ymm1, %ymm0, %ymm0
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX512VPOPCNT-NEXT: vmovq %xmm0, %rax
@@ -3441,17 +3607,21 @@ define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64>
; X86-AVX1-NEXT: vpaddb %xmm7, %xmm3, %xmm3
; X86-AVX1-NEXT: vpaddb %xmm6, %xmm3, %xmm3
; X86-AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm1[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm5, %xmm1, %xmm1
; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpunpckhqdq {{.*#+}} xmm5 = xmm2[1],xmm3[1]
-; X86-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
; X86-AVX1-NEXT: vpaddq %xmm5, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpunpckhqdq {{.*#+}} xmm3 = xmm0[1],xmm1[1]
-; X86-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT: vinsertf128 $1, %xmm3, %ymm1, %ymm1
; X86-AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; X86-AVX1-NEXT: vunpcklpd {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
; X86-AVX1-NEXT: movl %ebp, %esp
; X86-AVX1-NEXT: popl %ebp
; X86-AVX1-NEXT: retl
@@ -3517,13 +3687,13 @@ define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64>
; X64-AVX1-NEXT: vpaddb %xmm7, %xmm3, %xmm3
; X64-AVX1-NEXT: vpaddb %xmm6, %xmm3, %xmm3
; X64-AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm2, %xmm2
; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
-; X64-AVX1-NEXT: vpunpckhqdq {{.*#+}} xmm5 = xmm2[1],xmm3[1]
+; X64-AVX1-NEXT: vpunpckhqdq {{.*#+}} xmm4 = xmm2[1],xmm3[1]
; X64-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm3[0]
-; X64-AVX1-NEXT: vpaddq %xmm5, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpsadbw %xmm4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpaddq %xmm4, %xmm2, %xmm2
; X64-AVX1-NEXT: vpunpckhqdq {{.*#+}} xmm3 = xmm0[1],xmm1[1]
; X64-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; X64-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
@@ -3569,15 +3739,26 @@ define <4 x i64> @reduce_ctpop_v4i64_buildvector_v4i64(<4 x i64> %a0, <4 x i64>
; X86-AVX2-NEXT: vpshufb %ymm3, %ymm5, %ymm3
; X86-AVX2-NEXT: vpaddb %ymm7, %ymm3, %ymm3
; X86-AVX2-NEXT: vpsadbw %ymm6, %ymm3, %ymm3
-; X86-AVX2-NEXT: vperm2i128 {{.*#+}} ymm4 = ymm1[2,3],ymm3[2,3]
-; X86-AVX2-NEXT: vinserti128 $1, %xmm3, %ymm1, %ymm1
-; X86-AVX2-NEXT: vpaddq %ymm4, %ymm1, %ymm1
-; X86-AVX2-NEXT: vperm2i128 {{.*#+}} ymm3 = ymm0[2,3],ymm2[2,3]
-; X86-AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpaddq %ymm3, %ymm0, %ymm0
-; X86-AVX2-NEXT: vpunpckhqdq {{.*#+}} ymm2 = ymm0[1],ymm1[1],ymm0[3],ymm1[3]
-; X86-AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm1[0],ymm0[2],ymm1[2]
-; X86-AVX2-NEXT: vpaddq %ymm2, %ymm0, %ymm0
+; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm4
+; X86-AVX2-NEXT: vpaddq %xmm4, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm4, %xmm0, %xmm0
+; X86-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm4
+; X86-AVX2-NEXT: vpaddq %xmm4, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm4, %xmm1, %xmm1
+; X86-AVX2-NEXT: vextracti128 $1, %ymm2, %xmm4
+; X86-AVX2-NEXT: vpaddq %xmm4, %xmm2, %xmm2
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm4, %xmm2, %xmm2
+; X86-AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
+; X86-AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm3
+; X86-AVX2-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
+; X86-AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpunpcklqdq {{.*#+}} ymm0 = ymm0[0],ymm2[0],ymm0[2],ymm2[2]
+; X86-AVX2-NEXT: vpermq {{.*#+}} ymm0 = ymm0[0,2,1,3]
; X86-AVX2-NEXT: movl %ebp, %esp
; X86-AVX2-NEXT: popl %ebp
; X86-AVX2-NEXT: retl
@@ -4646,32 +4827,38 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; X64-SSE4-NEXT: paddq %xmm1, %xmm0
; X64-SSE4-NEXT: psadbw %xmm9, %xmm3
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm3, %xmm1
-; X64-SSE4-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm10 = xmm3[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm3, %xmm10
; X64-SSE4-NEXT: psadbw %xmm9, %xmm2
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm2, %xmm1
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm2, %xmm3
; X64-SSE4-NEXT: psadbw %xmm9, %xmm4
; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm4[2,3,2,3]
; X64-SSE4-NEXT: paddq %xmm4, %xmm2
-; X64-SSE4-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X64-SSE4-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
; X64-SSE4-NEXT: psadbw %xmm9, %xmm5
; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm5[2,3,2,3]
; X64-SSE4-NEXT: paddq %xmm5, %xmm1
; X64-SSE4-NEXT: psadbw %xmm9, %xmm6
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm6[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm6, %xmm2
-; X64-SSE4-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm4 = xmm6[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm6, %xmm4
; X64-SSE4-NEXT: psadbw %xmm9, %xmm7
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm2 = xmm7[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm7, %xmm2
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm5 = xmm7[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm7, %xmm5
; X64-SSE4-NEXT: psadbw %xmm9, %xmm8
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm3 = xmm8[2,3,2,3]
-; X64-SSE4-NEXT: paddq %xmm8, %xmm3
-; X64-SSE4-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X64-SSE4-NEXT: punpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm2[0]
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm6 = xmm8[2,3,2,3]
+; X64-SSE4-NEXT: paddq %xmm8, %xmm6
+; X64-SSE4-NEXT: movd %xmm10, %eax
+; X64-SSE4-NEXT: movd %xmm3, %ecx
+; X64-SSE4-NEXT: movd %xmm2, %edx
+; X64-SSE4-NEXT: movd %xmm4, %esi
+; X64-SSE4-NEXT: movd %xmm5, %edi
+; X64-SSE4-NEXT: movd %xmm6, %r8d
+; X64-SSE4-NEXT: pinsrd $1, %eax, %xmm0
+; X64-SSE4-NEXT: pinsrd $2, %ecx, %xmm0
+; X64-SSE4-NEXT: pinsrd $3, %edx, %xmm0
+; X64-SSE4-NEXT: pinsrd $1, %esi, %xmm1
+; X64-SSE4-NEXT: pinsrd $2, %edi, %xmm1
+; X64-SSE4-NEXT: pinsrd $3, %r8d, %xmm1
; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: reduce_ctpop_v4i64_buildvector_v8i32:
@@ -4713,65 +4900,66 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; X86-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
; X86-AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm5
+; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm5
+; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm2
; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm2, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm5, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpaddb %xmm0, %xmm2, %xmm0
; X86-AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-AVX1-NEXT: vmovdqa 8(%ebp), %xmm0
-; X86-AVX1-NEXT: vmovdqa 24(%ebp), %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm5
-; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
-; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm5, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm5
+; X86-AVX1-NEXT: vmovdqa 24(%ebp), %xmm5
+; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
+; X86-AVX1-NEXT: vpsrlw $4, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpand %xmm3, %xmm5, %xmm5
; X86-AVX1-NEXT: vpshufb %xmm5, %xmm4, %xmm5
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
; X86-AVX1-NEXT: vpaddb %xmm5, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
; X86-AVX1-NEXT: vmovdqa 56(%ebp), %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm6
+; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vmovdqa 40(%ebp), %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm6
+; X86-AVX1-NEXT: vpaddb %xmm6, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovdqa 40(%ebp), %xmm6
+; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
+; X86-AVX1-NEXT: vpsrlw $4, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpand %xmm3, %xmm6, %xmm6
; X86-AVX1-NEXT: vpshufb %xmm6, %xmm4, %xmm6
-; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm6, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm6
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpaddb %xmm0, %xmm6, %xmm0
+; X86-AVX1-NEXT: vmovdqa %xmm0, (%esp) # 16-byte Spill
; X86-AVX1-NEXT: vmovdqa 88(%ebp), %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm1
-; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vmovdqa 72(%ebp), %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm7
-; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
-; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm7, %xmm0, %xmm0
+; X86-AVX1-NEXT: vmovdqa 72(%ebp), %xmm7
+; X86-AVX1-NEXT: vpand %xmm3, %xmm7, %xmm1
; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
-; X86-AVX1-NEXT: vpaddb %xmm7, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsrlw $4, %xmm7, %xmm7
+; X86-AVX1-NEXT: vpand %xmm3, %xmm7, %xmm7
+; X86-AVX1-NEXT: vpshufb %xmm7, %xmm4, %xmm7
+; X86-AVX1-NEXT: vpaddb %xmm1, %xmm7, %xmm1
; X86-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm7
; X86-AVX1-NEXT: vmovdqa 120(%ebp), %xmm0
; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm1
@@ -4788,13 +4976,13 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
; X86-AVX1-NEXT: vpaddb %xmm2, %xmm1, %xmm1
; X86-AVX1-NEXT: vpaddb %xmm0, %xmm1, %xmm5
-; X86-AVX1-NEXT: vmovdqa 152(%ebp), %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm2
+; X86-AVX1-NEXT: vmovdqa 152(%ebp), %xmm1
+; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm2
; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
-; X86-AVX1-NEXT: vpsrlw $4, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpand %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpshufb %xmm0, %xmm4, %xmm0
-; X86-AVX1-NEXT: vpaddb %xmm2, %xmm0, %xmm1
+; X86-AVX1-NEXT: vpsrlw $4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpand %xmm3, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpshufb %xmm1, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpaddb %xmm2, %xmm1, %xmm1
; X86-AVX1-NEXT: vmovdqa 136(%ebp), %xmm2
; X86-AVX1-NEXT: vpand %xmm3, %xmm2, %xmm0
; X86-AVX1-NEXT: vpsrlw $4, %xmm2, %xmm2
@@ -4803,41 +4991,38 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; X86-AVX1-NEXT: vpshufb %xmm2, %xmm4, %xmm2
; X86-AVX1-NEXT: vpaddb %xmm0, %xmm2, %xmm0
; X86-AVX1-NEXT: vpaddb %xmm1, %xmm0, %xmm3
-; X86-AVX1-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpsadbw %xmm0, %xmm6, %xmm1
+; X86-AVX1-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %xmm4, %xmm0 # 16-byte Folded Reload
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %xmm4, %xmm1 # 16-byte Folded Reload
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; X86-AVX1-NEXT: vpaddq %xmm2, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw %xmm0, %xmm7, %xmm2
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm4, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm4 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X86-AVX1-NEXT: vpsadbw %xmm0, %xmm5, %xmm1
+; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-AVX1-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %xmm4, %xmm1 # 16-byte Folded Reload
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; X86-AVX1-NEXT: vpaddq %xmm2, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm2 # 16-byte Folded Reload
-; X86-AVX1-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm5 # 16-byte Folded Reload
-; X86-AVX1-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm6 # 16-byte Folded Reload
-; X86-AVX1-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %xmm0, %xmm7 # 16-byte Folded Reload
-; X86-AVX1-NEXT: vpsadbw %xmm0, %xmm3, %xmm0
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm7[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm7, %xmm1
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm6[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm6, %xmm3
-; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm5[2,3,2,3]
-; X86-AVX1-NEXT: vpaddq %xmm3, %xmm5, %xmm3
+; X86-AVX1-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %xmm4, %xmm2 # 16-byte Folded Reload
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm6, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X86-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm0[0],xmm1[0]
+; X86-AVX1-NEXT: vpsadbw (%esp), %xmm4, %xmm0 # 16-byte Folded Reload
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm7, %xmm2
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm6 = xmm2[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm6, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm5, %xmm2
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm5 = xmm2[2,3,2,3]
; X86-AVX1-NEXT: vpaddq %xmm5, %xmm2, %xmm2
+; X86-AVX1-NEXT: vpsadbw %xmm4, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; X86-AVX1-NEXT: vpaddq %xmm4, %xmm3, %xmm3
; X86-AVX1-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
-; X86-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm2[0],xmm1[0]
-; X86-AVX1-NEXT: vinsertf128 $1, %xmm4, %ymm0, %ymm2
-; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1]
-; X86-AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0
-; X86-AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm2[0,1,2,3,4,5],ymm0[6,7]
-; X86-AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm1[0,1,2,3],ymm0[4,5,6,7]
+; X86-AVX1-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; X86-AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
; X86-AVX1-NEXT: movl %ebp, %esp
; X86-AVX1-NEXT: popl %ebp
; X86-AVX1-NEXT: retl
@@ -5003,7 +5188,7 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; X86-AVX2-NEXT: pushl %ebp
; X86-AVX2-NEXT: movl %esp, %ebp
; X86-AVX2-NEXT: andl $-32, %esp
-; X86-AVX2-NEXT: subl $192, %esp
+; X86-AVX2-NEXT: subl $160, %esp
; X86-AVX2-NEXT: vmovdqa 40(%ebp), %ymm6
; X86-AVX2-NEXT: vmovdqa 8(%ebp), %ymm5
; X86-AVX2-NEXT: vpbroadcastb {{.*#+}} ymm3 = [15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15,15]
@@ -5022,110 +5207,103 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
; X86-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
; X86-AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm0
-; X86-AVX2-NEXT: vmovdqa %ymm0, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
+; X86-AVX2-NEXT: vmovdqa %ymm0, (%esp) # 32-byte Spill
; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm0
; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
; X86-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
-; X86-AVX2-NEXT: vpaddb %ymm0, %ymm2, %ymm0
-; X86-AVX2-NEXT: vmovdqa %ymm0, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
-; X86-AVX2-NEXT: vpand %ymm3, %ymm5, %ymm2
-; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
+; X86-AVX2-NEXT: vpaddb %ymm0, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpand %ymm3, %ymm5, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
; X86-AVX2-NEXT: vpsrlw $4, %ymm5, %ymm5
; X86-AVX2-NEXT: vpand %ymm3, %ymm5, %ymm5
; X86-AVX2-NEXT: vpshufb %ymm5, %ymm4, %ymm5
-; X86-AVX2-NEXT: vpaddb %ymm2, %ymm5, %ymm5
-; X86-AVX2-NEXT: vpand %ymm3, %ymm6, %ymm2
-; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
+; X86-AVX2-NEXT: vpaddb %ymm0, %ymm5, %ymm0
+; X86-AVX2-NEXT: vmovdqa %ymm0, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
+; X86-AVX2-NEXT: vpand %ymm3, %ymm6, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
; X86-AVX2-NEXT: vpsrlw $4, %ymm6, %ymm6
; X86-AVX2-NEXT: vpand %ymm3, %ymm6, %ymm6
; X86-AVX2-NEXT: vpshufb %ymm6, %ymm4, %ymm6
-; X86-AVX2-NEXT: vpaddb %ymm2, %ymm6, %ymm0
-; X86-AVX2-NEXT: vmovdqa %ymm0, (%esp) # 32-byte Spill
-; X86-AVX2-NEXT: vmovdqa 72(%ebp), %ymm2
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm7
+; X86-AVX2-NEXT: vpaddb %ymm0, %ymm6, %ymm6
+; X86-AVX2-NEXT: vmovdqa 72(%ebp), %ymm0
+; X86-AVX2-NEXT: vpand %ymm3, %ymm0, %ymm7
; X86-AVX2-NEXT: vpshufb %ymm7, %ymm4, %ymm7
-; X86-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
-; X86-AVX2-NEXT: vpaddb %ymm7, %ymm2, %ymm6
-; X86-AVX2-NEXT: vmovdqa 104(%ebp), %ymm2
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm1
+; X86-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
+; X86-AVX2-NEXT: vpaddb %ymm7, %ymm0, %ymm7
+; X86-AVX2-NEXT: vmovdqa 104(%ebp), %ymm0
+; X86-AVX2-NEXT: vpand %ymm3, %ymm0, %ymm1
; X86-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
-; X86-AVX2-NEXT: vpsrlw $4, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpand %ymm3, %ymm2, %ymm2
-; X86-AVX2-NEXT: vpshufb %ymm2, %ymm4, %ymm2
-; X86-AVX2-NEXT: vpaddb %ymm1, %ymm2, %ymm7
+; X86-AVX2-NEXT: vpsrlw $4, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpand %ymm3, %ymm0, %ymm0
+; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
+; X86-AVX2-NEXT: vpaddb %ymm1, %ymm0, %ymm0
+; X86-AVX2-NEXT: vmovdqa %ymm0, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
; X86-AVX2-NEXT: vmovdqa 136(%ebp), %ymm1
; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm0
; X86-AVX2-NEXT: vpsrlw $4, %ymm1, %ymm1
; X86-AVX2-NEXT: vpand %ymm3, %ymm1, %ymm1
; X86-AVX2-NEXT: vpshufb %ymm0, %ymm4, %ymm0
; X86-AVX2-NEXT: vpshufb %ymm1, %ymm4, %ymm1
-; X86-AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm0
-; X86-AVX2-NEXT: vmovdqa %ymm0, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
-; X86-AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm0, %ymm1 # 32-byte Folded Reload
-; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm0, %ymm2 # 32-byte Folded Reload
-; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm0, %ymm3 # 32-byte Folded Reload
-; X86-AVX2-NEXT: vpsadbw %ymm0, %ymm5, %ymm5
-; X86-AVX2-NEXT: vpsadbw (%esp), %ymm0, %ymm4 # 32-byte Folded Reload
-; X86-AVX2-NEXT: vpsadbw %ymm0, %ymm6, %ymm6
-; X86-AVX2-NEXT: vpsadbw %ymm0, %ymm7, %ymm7
+; X86-AVX2-NEXT: vpaddb %ymm0, %ymm1, %ymm5
+; X86-AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm1, %ymm3 # 32-byte Folded Reload
+; X86-AVX2-NEXT: vpsadbw (%esp), %ymm1, %ymm0 # 32-byte Folded Reload
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm1, %ymm4 # 32-byte Folded Reload
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm6, %ymm6
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm7, %ymm7
; X86-AVX2-NEXT: vmovdqa %ymm7, (%esp) # 32-byte Spill
-; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm0, %ymm0 # 32-byte Folded Reload
-; X86-AVX2-NEXT: vmovdqa %ymm0, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
-; X86-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm0
-; X86-AVX2-NEXT: vpaddq %xmm0, %xmm1, %xmm0
-; X86-AVX2-NEXT: vmovdqa %xmm0, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-AVX2-NEXT: vextracti128 $1, %ymm2, %xmm0
-; X86-AVX2-NEXT: vpaddq %xmm0, %xmm2, %xmm0
-; X86-AVX2-NEXT: vextracti128 $1, %ymm3, %xmm1
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm3, %xmm3
-; X86-AVX2-NEXT: vextracti128 $1, %ymm5, %xmm2
-; X86-AVX2-NEXT: vpaddq %xmm2, %xmm5, %xmm7
-; X86-AVX2-NEXT: vextracti128 $1, %ymm4, %xmm2
-; X86-AVX2-NEXT: vpaddq %xmm2, %xmm4, %xmm1
-; X86-AVX2-NEXT: vmovdqa %xmm1, {{[-0-9]+}}(%e{{[sb]}}p) # 16-byte Spill
-; X86-AVX2-NEXT: vextracti128 $1, %ymm6, %xmm2
-; X86-AVX2-NEXT: vpaddq %xmm2, %xmm6, %xmm6
-; X86-AVX2-NEXT: vmovdqa (%esp), %ymm1 # 32-byte Reload
-; X86-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; X86-AVX2-NEXT: vpaddq %xmm2, %xmm1, %xmm4
-; X86-AVX2-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %ymm1 # 32-byte Reload
-; X86-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm2
-; X86-AVX2-NEXT: vpaddq %xmm2, %xmm1, %xmm2
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm5, %xmm0, %xmm1
-; X86-AVX2-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm0 # 16-byte Reload
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm5, %xmm0, %xmm5
-; X86-AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm5[0],xmm1[0],xmm5[1],xmm1[1]
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm7[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm5, %xmm7, %xmm5
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm3[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm7, %xmm3, %xmm1
-; X86-AVX2-NEXT: vpbroadcastd %xmm5, %xmm5
-; X86-AVX2-NEXT: vpbroadcastd %xmm1, %xmm1
-; X86-AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
-; X86-AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3]
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm6[2,3,2,3]
-; X86-AVX2-NEXT: vpaddq %xmm1, %xmm6, %xmm1
-; X86-AVX2-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %xmm3 # 16-byte Reload
+; X86-AVX2-NEXT: vpsadbw {{[-0-9]+}}(%e{{[sb]}}p), %ymm1, %ymm7 # 32-byte Folded Reload
+; X86-AVX2-NEXT: vmovdqa %ymm7, {{[-0-9]+}}(%e{{[sb]}}p) # 32-byte Spill
+; X86-AVX2-NEXT: vpsadbw %ymm1, %ymm5, %ymm1
+; X86-AVX2-NEXT: vextracti128 $1, %ymm3, %xmm5
+; X86-AVX2-NEXT: vpaddq %xmm5, %xmm3, %xmm3
; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
; X86-AVX2-NEXT: vpaddq %xmm5, %xmm3, %xmm3
-; X86-AVX2-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm3[0],xmm1[0],xmm3[1],xmm1[1]
+; X86-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm5
+; X86-AVX2-NEXT: vpaddq %xmm5, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm0[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm5, %xmm0, %xmm0
+; X86-AVX2-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm3[0],xmm0[0],xmm3[1],xmm0[1]
+; X86-AVX2-NEXT: vextracti128 $1, %ymm2, %xmm3
+; X86-AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[2,3,2,3]
; X86-AVX2-NEXT: vpaddq %xmm3, %xmm2, %xmm2
-; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm3 = xmm4[2,3,2,3]
+; X86-AVX2-NEXT: vextracti128 $1, %ymm4, %xmm3
; X86-AVX2-NEXT: vpaddq %xmm3, %xmm4, %xmm3
-; X86-AVX2-NEXT: vpbroadcastd %xmm2, %xmm2
-; X86-AVX2-NEXT: vpbroadcastd %xmm3, %xmm3
-; X86-AVX2-NEXT: vpunpckldq {{.*#+}} xmm2 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X86-AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1
-; X86-AVX2-NEXT: vinserti128 $1, %xmm2, %ymm0, %ymm2
-; X86-AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm1[0,1,2,3,4,5],ymm2[6,7]
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm4, %xmm3, %xmm4
+; X86-AVX2-NEXT: vextracti128 $1, %ymm6, %xmm3
+; X86-AVX2-NEXT: vpaddq %xmm3, %xmm6, %xmm3
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm5, %xmm3, %xmm6
+; X86-AVX2-NEXT: vmovdqa (%esp), %ymm5 # 32-byte Reload
+; X86-AVX2-NEXT: vextracti128 $1, %ymm5, %xmm3
+; X86-AVX2-NEXT: vpaddq %xmm3, %xmm5, %xmm3
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm5 = xmm3[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm5, %xmm3, %xmm3
+; X86-AVX2-NEXT: vmovdqa {{[-0-9]+}}(%e{{[sb]}}p), %ymm7 # 32-byte Reload
+; X86-AVX2-NEXT: vextracti128 $1, %ymm7, %xmm5
+; X86-AVX2-NEXT: vpaddq %xmm5, %xmm7, %xmm5
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm5[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm7, %xmm5, %xmm5
+; X86-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm7
+; X86-AVX2-NEXT: vpaddq %xmm7, %xmm1, %xmm1
+; X86-AVX2-NEXT: vpshufd {{.*#+}} xmm7 = xmm1[2,3,2,3]
+; X86-AVX2-NEXT: vpaddq %xmm7, %xmm1, %xmm1
+; X86-AVX2-NEXT: vinserti128 $1, %xmm1, %ymm5, %ymm1
+; X86-AVX2-NEXT: vinserti128 $1, %xmm3, %ymm6, %ymm3
+; X86-AVX2-NEXT: vpbroadcastq {{.*#+}} ymm5 = [0,4,0,4,0,4,0,4]
+; X86-AVX2-NEXT: vpermd %ymm1, %ymm5, %ymm1
+; X86-AVX2-NEXT: vpermd %ymm3, %ymm5, %ymm3
+; X86-AVX2-NEXT: vpblendd {{.*#+}} ymm1 = ymm3[0,1,2,3,4,5],ymm1[6,7]
+; X86-AVX2-NEXT: vinserti128 $1, %xmm4, %ymm2, %ymm2
+; X86-AVX2-NEXT: vpermd %ymm2, %ymm5, %ymm2
+; X86-AVX2-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3]
; X86-AVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm1[4,5,6,7]
; X86-AVX2-NEXT: movl %ebp, %esp
; X86-AVX2-NEXT: popl %ebp
@@ -5196,42 +5374,42 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; X64-AVX2-NEXT: vextracti128 $1, %ymm0, %xmm8
; X64-AVX2-NEXT: vpaddq %xmm0, %xmm8, %xmm0
; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm9
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm9, %xmm1
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm9 = xmm1[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm2, %xmm10
-; X64-AVX2-NEXT: vpaddq %xmm2, %xmm10, %xmm2
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm10 = xmm2[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm3, %xmm11
-; X64-AVX2-NEXT: vpaddq %xmm3, %xmm11, %xmm3
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm11 = xmm3[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm4, %xmm12
-; X64-AVX2-NEXT: vpaddq %xmm4, %xmm12, %xmm4
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm12 = xmm4[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm5, %xmm13
-; X64-AVX2-NEXT: vpaddq %xmm5, %xmm13, %xmm5
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm13 = xmm5[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm6, %xmm14
-; X64-AVX2-NEXT: vpaddq %xmm6, %xmm14, %xmm6
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm14 = xmm6[2,3,2,3]
-; X64-AVX2-NEXT: vextracti128 $1, %ymm7, %xmm15
-; X64-AVX2-NEXT: vpaddq %xmm7, %xmm15, %xmm7
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm15 = xmm7[2,3,2,3]
; X64-AVX2-NEXT: vpaddq %xmm0, %xmm8, %xmm0
-; X64-AVX2-NEXT: vpaddq %xmm1, %xmm9, %xmm1
+; X64-AVX2-NEXT: vextracti128 $1, %ymm1, %xmm8
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm8, %xmm1
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm1[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm1, %xmm8, %xmm1
+; X64-AVX2-NEXT: vextracti128 $1, %ymm2, %xmm8
+; X64-AVX2-NEXT: vpaddq %xmm2, %xmm8, %xmm2
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm2[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm2, %xmm8, %xmm2
+; X64-AVX2-NEXT: vextracti128 $1, %ymm3, %xmm8
+; X64-AVX2-NEXT: vpaddq %xmm3, %xmm8, %xmm3
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm3[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm3, %xmm8, %xmm3
+; X64-AVX2-NEXT: vextracti128 $1, %ymm4, %xmm8
+; X64-AVX2-NEXT: vpaddq %xmm4, %xmm8, %xmm4
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm4[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm4, %xmm8, %xmm4
+; X64-AVX2-NEXT: vextracti128 $1, %ymm5, %xmm8
+; X64-AVX2-NEXT: vpaddq %xmm5, %xmm8, %xmm5
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm5, %xmm8, %xmm5
+; X64-AVX2-NEXT: vextracti128 $1, %ymm6, %xmm8
+; X64-AVX2-NEXT: vpaddq %xmm6, %xmm8, %xmm6
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm6, %xmm8, %xmm6
+; X64-AVX2-NEXT: vextracti128 $1, %ymm7, %xmm8
+; X64-AVX2-NEXT: vpaddq %xmm7, %xmm8, %xmm7
+; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[2,3,2,3]
+; X64-AVX2-NEXT: vpaddq %xmm7, %xmm8, %xmm7
; X64-AVX2-NEXT: vmovd %xmm1, %eax
-; X64-AVX2-NEXT: vpaddq %xmm2, %xmm10, %xmm1
-; X64-AVX2-NEXT: vmovd %xmm1, %ecx
-; X64-AVX2-NEXT: vpaddq %xmm3, %xmm11, %xmm1
-; X64-AVX2-NEXT: vmovd %xmm1, %edx
-; X64-AVX2-NEXT: vpaddq %xmm4, %xmm12, %xmm1
-; X64-AVX2-NEXT: vpaddq %xmm5, %xmm13, %xmm2
-; X64-AVX2-NEXT: vmovd %xmm2, %esi
-; X64-AVX2-NEXT: vpaddq %xmm6, %xmm14, %xmm2
-; X64-AVX2-NEXT: vmovd %xmm2, %edi
-; X64-AVX2-NEXT: vpaddq %xmm7, %xmm15, %xmm2
-; X64-AVX2-NEXT: vmovd %xmm2, %r8d
-; X64-AVX2-NEXT: vpinsrd $1, %esi, %xmm1, %xmm1
+; X64-AVX2-NEXT: vmovd %xmm2, %ecx
+; X64-AVX2-NEXT: vmovd %xmm3, %edx
+; X64-AVX2-NEXT: vmovd %xmm5, %esi
+; X64-AVX2-NEXT: vmovd %xmm6, %edi
+; X64-AVX2-NEXT: vmovd %xmm7, %r8d
+; X64-AVX2-NEXT: vpinsrd $1, %esi, %xmm4, %xmm1
; X64-AVX2-NEXT: vpinsrd $2, %edi, %xmm1, %xmm1
; X64-AVX2-NEXT: vpinsrd $3, %r8d, %xmm1, %xmm1
; X64-AVX2-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
@@ -5302,28 +5480,51 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; AVX512VL-NEXT: vpshufb %ymm7, %ymm10, %ymm7
; AVX512VL-NEXT: vpaddb %ymm7, %ymm11, %ymm7
; AVX512VL-NEXT: vpsadbw %ymm7, %ymm9, %ymm7
-; AVX512VL-NEXT: vpmovqb %ymm0, %xmm0
-; AVX512VL-NEXT: vpmovqb %ymm1, %xmm1
-; AVX512VL-NEXT: vpmovqb %ymm2, %xmm2
-; AVX512VL-NEXT: vpmovqb %ymm3, %xmm3
-; AVX512VL-NEXT: vpmovqb %ymm4, %xmm4
-; AVX512VL-NEXT: vpmovqb %ymm5, %xmm5
-; AVX512VL-NEXT: vpmovqb %ymm6, %xmm6
-; AVX512VL-NEXT: vpmovqb %ymm7, %xmm7
-; AVX512VL-NEXT: vinserti128 $1, %xmm5, %ymm4, %ymm4
-; AVX512VL-NEXT: vinserti128 $1, %xmm7, %ymm6, %ymm5
-; AVX512VL-NEXT: vinserti64x4 $1, %ymm4, %zmm5, %zmm4
-; AVX512VL-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX512VL-NEXT: vpsadbw %zmm5, %zmm4, %zmm4
-; AVX512VL-NEXT: vpmovsxbd {{.*#+}} ymm6 = [8,12,8,12,8,12,0,4]
-; AVX512VL-NEXT: vpermd %zmm4, %zmm6, %zmm4
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm8
+; AVX512VL-NEXT: vpaddq %xmm0, %xmm8, %xmm0
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm0, %xmm8, %xmm0
+; AVX512VL-NEXT: vextracti128 $1, %ymm1, %xmm8
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm8, %xmm1
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm8 = xmm1[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm8, %xmm1
+; AVX512VL-NEXT: vextracti128 $1, %ymm2, %xmm8
+; AVX512VL-NEXT: vpaddq %xmm2, %xmm8, %xmm2
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm8 = xmm2[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm2, %xmm8, %xmm2
+; AVX512VL-NEXT: vextracti128 $1, %ymm3, %xmm8
+; AVX512VL-NEXT: vpaddq %xmm3, %xmm8, %xmm3
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm8 = xmm3[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm3, %xmm8, %xmm3
+; AVX512VL-NEXT: vextracti128 $1, %ymm4, %xmm8
+; AVX512VL-NEXT: vpaddq %xmm4, %xmm8, %xmm4
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm8 = xmm4[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm4, %xmm8, %xmm4
+; AVX512VL-NEXT: vextracti128 $1, %ymm5, %xmm8
+; AVX512VL-NEXT: vpaddq %xmm5, %xmm8, %xmm5
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm5, %xmm8, %xmm5
+; AVX512VL-NEXT: vextracti128 $1, %ymm6, %xmm8
+; AVX512VL-NEXT: vpaddq %xmm6, %xmm8, %xmm6
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm6, %xmm8, %xmm6
+; AVX512VL-NEXT: vextracti128 $1, %ymm7, %xmm8
+; AVX512VL-NEXT: vpaddq %xmm7, %xmm8, %xmm7
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm7, %xmm8, %xmm7
+; AVX512VL-NEXT: vmovd %xmm1, %eax
+; AVX512VL-NEXT: vmovd %xmm2, %ecx
+; AVX512VL-NEXT: vmovd %xmm3, %edx
+; AVX512VL-NEXT: vmovd %xmm5, %esi
+; AVX512VL-NEXT: vmovd %xmm6, %edi
+; AVX512VL-NEXT: vmovd %xmm7, %r8d
+; AVX512VL-NEXT: vpinsrd $1, %esi, %xmm4, %xmm1
+; AVX512VL-NEXT: vpinsrd $2, %edi, %xmm1, %xmm1
+; AVX512VL-NEXT: vpinsrd $3, %r8d, %xmm1, %xmm1
+; AVX512VL-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX512VL-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
+; AVX512VL-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512VL-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm1
-; AVX512VL-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512VL-NEXT: vpsadbw %zmm5, %zmm0, %zmm0
-; AVX512VL-NEXT: vpmovsxbd {{.*#+}} xmm1 = [8,12,0,4]
-; AVX512VL-NEXT: vpermd %zmm0, %zmm1, %zmm0
-; AVX512VL-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
; AVX512VL-NEXT: retq
;
; AVX512VPOPCNT-LABEL: reduce_ctpop_v4i64_buildvector_v8i32:
@@ -5336,28 +5537,51 @@ define <8 x i32> @reduce_ctpop_v4i64_buildvector_v8i32(<4 x i64> %a0, <4 x i64>
; AVX512VPOPCNT-NEXT: vpopcntq %ymm5, %ymm5
; AVX512VPOPCNT-NEXT: vpopcntq %ymm6, %ymm6
; AVX512VPOPCNT-NEXT: vpopcntq %ymm7, %ymm7
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm0, %xmm0
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm1, %xmm1
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm2, %xmm2
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm3, %xmm3
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm4, %xmm4
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm5, %xmm5
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm6, %xmm6
-; AVX512VPOPCNT-NEXT: vpmovqb %ymm7, %xmm7
-; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm5, %ymm4, %ymm4
-; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm7, %ymm6, %ymm5
-; AVX512VPOPCNT-NEXT: vinserti64x4 $1, %ymm4, %zmm5, %zmm4
-; AVX512VPOPCNT-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX512VPOPCNT-NEXT: vpsadbw %zmm5, %zmm4, %zmm4
-; AVX512VPOPCNT-NEXT: vpmovsxbd {{.*#+}} ymm6 = [8,12,8,12,8,12,0,4]
-; AVX512VPOPCNT-NEXT: vpermd %zmm4, %zmm6, %zmm4
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm0, %xmm8
+; AVX512VPOPCNT-NEXT: vpaddq %xmm0, %xmm8, %xmm0
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm0, %xmm8, %xmm0
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm1, %xmm8
+; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm8, %xmm1
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm8 = xmm1[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm1, %xmm8, %xmm1
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm2, %xmm8
+; AVX512VPOPCNT-NEXT: vpaddq %xmm2, %xmm8, %xmm2
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm8 = xmm2[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm2, %xmm8, %xmm2
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm3, %xmm8
+; AVX512VPOPCNT-NEXT: vpaddq %xmm3, %xmm8, %xmm3
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm8 = xmm3[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm3, %xmm8, %xmm3
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm4, %xmm8
+; AVX512VPOPCNT-NEXT: vpaddq %xmm4, %xmm8, %xmm4
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm8 = xmm4[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm4, %xmm8, %xmm4
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm5, %xmm8
+; AVX512VPOPCNT-NEXT: vpaddq %xmm5, %xmm8, %xmm5
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm8 = xmm5[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm5, %xmm8, %xmm5
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm6, %xmm8
+; AVX512VPOPCNT-NEXT: vpaddq %xmm6, %xmm8, %xmm6
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm8 = xmm6[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm6, %xmm8, %xmm6
+; AVX512VPOPCNT-NEXT: vextracti128 $1, %ymm7, %xmm8
+; AVX512VPOPCNT-NEXT: vpaddq %xmm7, %xmm8, %xmm7
+; AVX512VPOPCNT-NEXT: vpshufd {{.*#+}} xmm8 = xmm7[2,3,2,3]
+; AVX512VPOPCNT-NEXT: vpaddq %xmm7, %xmm8, %xmm7
+; AVX512VPOPCNT-NEXT: vmovd %xmm1, %eax
+; AVX512VPOPCNT-NEXT: vmovd %xmm2, %ecx
+; AVX512VPOPCNT-NEXT: vmovd %xmm3, %edx
+; AVX512VPOPCNT-NEXT: vmovd %xmm5, %esi
+; AVX512VPOPCNT-NEXT: vmovd %xmm6, %edi
+; AVX512VPOPCNT-NEXT: vmovd %xmm7, %r8d
+; AVX512VPOPCNT-NEXT: vpinsrd $1, %esi, %xmm4, %xmm1
+; AVX512VPOPCNT-NEXT: vpinsrd $2, %edi, %xmm1, %xmm1
+; AVX512VPOPCNT-NEXT: vpinsrd $3, %r8d, %xmm1, %xmm1
+; AVX512VPOPCNT-NEXT: vpinsrd $1, %eax, %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpinsrd $2, %ecx, %xmm0, %xmm0
+; AVX512VPOPCNT-NEXT: vpinsrd $3, %edx, %xmm0, %xmm0
; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
-; AVX512VPOPCNT-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm1
-; AVX512VPOPCNT-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0
-; AVX512VPOPCNT-NEXT: vpsadbw %zmm5, %zmm0, %zmm0
-; AVX512VPOPCNT-NEXT: vpmovsxbd {{.*#+}} xmm1 = [8,12,0,4]
-; AVX512VPOPCNT-NEXT: vpermd %zmm0, %zmm1, %zmm0
-; AVX512VPOPCNT-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3],ymm4[4,5,6,7]
; AVX512VPOPCNT-NEXT: retq
%p0 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a0)
%p1 = tail call <4 x i64> @llvm.ctpop.v4i64(<4 x i64> %a1)
diff --git a/llvm/test/CodeGen/X86/vector-trunc.ll b/llvm/test/CodeGen/X86/vector-trunc.ll
index 46f770a349d96..8c24fed3d42d7 100644
--- a/llvm/test/CodeGen/X86/vector-trunc.ll
+++ b/llvm/test/CodeGen/X86/vector-trunc.ll
@@ -1946,11 +1946,12 @@ define i16 @PR66194(i8 %q) {
; SSE2-SSSE3-NEXT: pinsrw $7, %ecx, %xmm0
; SSE2-SSSE3-NEXT: pcmpeqd %xmm1, %xmm1
; SSE2-SSSE3-NEXT: psubw %xmm1, %xmm0
-; SSE2-SSSE3-NEXT: packuswb %xmm0, %xmm0
; SSE2-SSSE3-NEXT: pxor %xmm1, %xmm1
; SSE2-SSSE3-NEXT: psadbw %xmm0, %xmm1
-; SSE2-SSSE3-NEXT: movd %xmm1, %eax
-; SSE2-SSSE3-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-SSSE3-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-SSSE3-NEXT: paddq %xmm1, %xmm0
+; SSE2-SSSE3-NEXT: movq %xmm0, %rax
+; SSE2-SSSE3-NEXT: # kill: def $ax killed $ax killed $rax
; SSE2-SSSE3-NEXT: retq
;
; SSE41-LABEL: PR66194:
@@ -1970,11 +1971,12 @@ define i16 @PR66194(i8 %q) {
; SSE41-NEXT: pinsrb $14, %ecx, %xmm0
; SSE41-NEXT: pcmpeqd %xmm1, %xmm1
; SSE41-NEXT: psubw %xmm1, %xmm0
-; SSE41-NEXT: packuswb %xmm0, %xmm0
; SSE41-NEXT: pxor %xmm1, %xmm1
; SSE41-NEXT: psadbw %xmm0, %xmm1
-; SSE41-NEXT: movd %xmm1, %eax
-; SSE41-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE41-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE41-NEXT: paddq %xmm1, %xmm0
+; SSE41-NEXT: movq %xmm0, %rax
+; SSE41-NEXT: # kill: def $ax killed $ax killed $rax
; SSE41-NEXT: retq
;
; AVX1-LABEL: PR66194:
@@ -1991,16 +1993,13 @@ define i16 @PR66194(i8 %q) {
; AVX1-NEXT: vpinsrb $10, %eax, %xmm0, %xmm0
; AVX1-NEXT: vpinsrb $12, %eax, %xmm0, %xmm0
; AVX1-NEXT: vpinsrb $14, %ecx, %xmm0, %xmm0
-; AVX1-NEXT: vpcmpeqd %xmm1, %xmm1, %xmm1
-; AVX1-NEXT: vpsubw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddb {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 # [1,1,1,1,1,1,1,1]
+; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX1-NEXT: vpsadbw %xmm1, %xmm0, %xmm0
; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX1-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX1-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX1-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vmovd %xmm0, %eax
-; AVX1-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vmovq %xmm0, %rax
+; AVX1-NEXT: # kill: def $ax killed $ax killed $rax
; AVX1-NEXT: retq
;
; AVX2-LABEL: PR66194:
@@ -2010,50 +2009,138 @@ define i16 @PR66194(i8 %q) {
; AVX2-NEXT: testb %dil, %dil
; AVX2-NEXT: setne %al
; AVX2-NEXT: sete %cl
-; AVX2-NEXT: vmovd %eax, %xmm0
-; AVX2-NEXT: vpinsrw $1, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $3, %eax, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $4, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $5, %eax, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $6, %eax, %xmm0, %xmm0
-; AVX2-NEXT: vpinsrw $7, %ecx, %xmm0, %xmm0
-; AVX2-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm0 = [1,1,1,1,1,1,1,1]
+; AVX2-NEXT: vmovd %eax, %xmm1
+; AVX2-NEXT: vpinsrw $1, %ecx, %xmm1, %xmm1
+; AVX2-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1
+; AVX2-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
+; AVX2-NEXT: vpinsrw $4, %ecx, %xmm1, %xmm1
+; AVX2-NEXT: vpinsrw $5, %eax, %xmm1, %xmm1
+; AVX2-NEXT: vpinsrw $6, %eax, %xmm1, %xmm1
+; AVX2-NEXT: vpinsrw $7, %ecx, %xmm1, %xmm1
+; AVX2-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vmovd %xmm0, %eax
-; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512-LABEL: PR66194:
-; AVX512: # %bb.0: # %entry
-; AVX512-NEXT: xorl %eax, %eax
-; AVX512-NEXT: xorl %ecx, %ecx
-; AVX512-NEXT: testb %dil, %dil
-; AVX512-NEXT: setne %al
-; AVX512-NEXT: sete %cl
-; AVX512-NEXT: vmovd %eax, %xmm0
-; AVX512-NEXT: vpinsrw $1, %ecx, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $2, %eax, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $3, %eax, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $4, %ecx, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $5, %eax, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $6, %eax, %xmm0, %xmm0
-; AVX512-NEXT: vpinsrw $7, %ecx, %xmm0, %xmm0
-; AVX512-NEXT: vpaddw {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512-NEXT: vpaddw %xmm1, %xmm0, %xmm0
-; AVX512-NEXT: vmovd %xmm0, %eax
-; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512-NEXT: retq
+; AVX512F-LABEL: PR66194:
+; AVX512F: # %bb.0: # %entry
+; AVX512F-NEXT: xorl %eax, %eax
+; AVX512F-NEXT: xorl %ecx, %ecx
+; AVX512F-NEXT: testb %dil, %dil
+; AVX512F-NEXT: setne %al
+; AVX512F-NEXT: sete %cl
+; AVX512F-NEXT: vpbroadcastw {{.*#+}} xmm0 = [1,1,1,1,1,1,1,1]
+; AVX512F-NEXT: vmovd %eax, %xmm1
+; AVX512F-NEXT: vpinsrw $1, %ecx, %xmm1, %xmm1
+; AVX512F-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1
+; AVX512F-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
+; AVX512F-NEXT: vpinsrw $4, %ecx, %xmm1, %xmm1
+; AVX512F-NEXT: vpinsrw $5, %eax, %xmm1, %xmm1
+; AVX512F-NEXT: vpinsrw $6, %eax, %xmm1, %xmm1
+; AVX512F-NEXT: vpinsrw $7, %ecx, %xmm1, %xmm1
+; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512F-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512F-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512F-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512F-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512F-NEXT: vmovq %xmm0, %rax
+; AVX512F-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
+;
+; AVX512VL-LABEL: PR66194:
+; AVX512VL: # %bb.0: # %entry
+; AVX512VL-NEXT: xorl %eax, %eax
+; AVX512VL-NEXT: xorl %ecx, %ecx
+; AVX512VL-NEXT: testb %dil, %dil
+; AVX512VL-NEXT: setne %al
+; AVX512VL-NEXT: sete %cl
+; AVX512VL-NEXT: vpbroadcastd {{.*#+}} xmm0 = [1,1,1,1,1,1,1,1]
+; AVX512VL-NEXT: vmovd %eax, %xmm1
+; AVX512VL-NEXT: vpinsrw $1, %ecx, %xmm1, %xmm1
+; AVX512VL-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1
+; AVX512VL-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
+; AVX512VL-NEXT: vpinsrw $4, %ecx, %xmm1, %xmm1
+; AVX512VL-NEXT: vpinsrw $5, %eax, %xmm1, %xmm1
+; AVX512VL-NEXT: vpinsrw $6, %eax, %xmm1, %xmm1
+; AVX512VL-NEXT: vpinsrw $7, %ecx, %xmm1, %xmm1
+; AVX512VL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512VL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512VL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512VL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512VL-NEXT: vmovq %xmm0, %rax
+; AVX512VL-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX512VL-NEXT: vzeroupper
+; AVX512VL-NEXT: retq
+;
+; AVX512BW-LABEL: PR66194:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: xorl %eax, %eax
+; AVX512BW-NEXT: xorl %ecx, %ecx
+; AVX512BW-NEXT: testb %dil, %dil
+; AVX512BW-NEXT: setne %al
+; AVX512BW-NEXT: sete %cl
+; AVX512BW-NEXT: vpbroadcastw {{.*#+}} xmm0 = [1,1,1,1,1,1,1,1]
+; AVX512BW-NEXT: vmovd %eax, %xmm1
+; AVX512BW-NEXT: vpinsrw $1, %ecx, %xmm1, %xmm1
+; AVX512BW-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1
+; AVX512BW-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
+; AVX512BW-NEXT: vpinsrw $4, %ecx, %xmm1, %xmm1
+; AVX512BW-NEXT: vpinsrw $5, %eax, %xmm1, %xmm1
+; AVX512BW-NEXT: vpinsrw $6, %eax, %xmm1, %xmm1
+; AVX512BW-NEXT: vpinsrw $7, %ecx, %xmm1, %xmm1
+; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovq %xmm0, %rax
+; AVX512BW-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+;
+; AVX512BWVL-LABEL: PR66194:
+; AVX512BWVL: # %bb.0: # %entry
+; AVX512BWVL-NEXT: xorl %eax, %eax
+; AVX512BWVL-NEXT: xorl %ecx, %ecx
+; AVX512BWVL-NEXT: testb %dil, %dil
+; AVX512BWVL-NEXT: setne %al
+; AVX512BWVL-NEXT: sete %cl
+; AVX512BWVL-NEXT: vpbroadcastw {{.*#+}} xmm0 = [1,1,1,1,1,1,1,1]
+; AVX512BWVL-NEXT: vmovd %eax, %xmm1
+; AVX512BWVL-NEXT: vpinsrw $1, %ecx, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpinsrw $2, %eax, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpinsrw $3, %eax, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpinsrw $4, %ecx, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpinsrw $5, %eax, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpinsrw $6, %eax, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpinsrw $7, %ecx, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BWVL-NEXT: vpsadbw %ymm1, %ymm0, %ymm0
+; AVX512BWVL-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BWVL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BWVL-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512BWVL-NEXT: vmovq %xmm0, %rax
+; AVX512BWVL-NEXT: # kill: def $ax killed $ax killed $rax
+; AVX512BWVL-NEXT: vzeroupper
+; AVX512BWVL-NEXT: retq
entry:
%cmp12.i.13 = icmp ne i8 %q, 0
%cond.i15.13 = zext i1 %cmp12.i.13 to i16
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
index acddf254c2998..752c20331a2a5 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
@@ -11,12 +11,8 @@ target datalayout = "e-m:e-i64:64-f80:128-n8:16:32:64-S128"
define i32 @add_v4i32(ptr %p) {
; CHECK-LABEL: @add_v4i32(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
-; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <4 x i32> [[TMP1]], <4 x i32> poison, <4 x i32> <i32 2, i32 3, i32 poison, i32 poison>
-; CHECK-NEXT: [[BIN_RDX:%.*]] = add <4 x i32> [[TMP1]], [[RDX_SHUF]]
-; CHECK-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <4 x i32> [[BIN_RDX]], <4 x i32> poison, <4 x i32> <i32 1, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[BIN_RDX4:%.*]] = add <4 x i32> [[BIN_RDX]], [[RDX_SHUF3]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <4 x i32> [[BIN_RDX4]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x i32>, ptr [[P:%.*]], align 4, !tbaa [[TBAA0:![0-9]+]]
+; CHECK-NEXT: [[TMP2:%.*]] = tail call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP0]])
; CHECK-NEXT: ret i32 [[TMP2]]
;
entry:
More information about the llvm-commits
mailing list