[llvm] [X86] Add ISD::VECREDUCE_MUL lowering instead of matching in DAG (PR #207593)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Jul 6 02:03:23 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/207593
>From 0251e4c6fe6282a66fe880e4e88d27aea3dcc6fb Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Sun, 5 Jul 2026 16:38:08 +0100
Subject: [PATCH 1/2] [X86] Add ISD::VECREDUCE_MUL lowering instead of matching
in DAG
Add X86 support for ISD::VECREDUCE_MUL and remove it from combineArithReduction - most of the change is refactoring the custom promotion of vXi8 -> vXi16.
Another part of #194621
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 97 ++-
.../lib/Target/X86/X86TargetTransformInfo.cpp | 1 +
.../X86/avx512-intrinsics-fast-isel.ll | 6 +-
llvm/test/CodeGen/X86/vector-reduce-mul.ll | 824 ++++++++----------
.../X86/vector-reductions-expanded.ll | 10 +-
5 files changed, 437 insertions(+), 501 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b33b01f804c6b..8888070f9234a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1180,6 +1180,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// SSE41 can use PHMINPOS to perform v16i8/v8i16 minmax reductions.
// Fallback to ReplaceNodeResults for vXi64 reductions on 32-bit targets.
for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::i64}) {
+ setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
@@ -1571,6 +1572,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
@@ -2047,6 +2049,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
@@ -2813,6 +2816,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
ISD::ANY_EXTEND_VECTOR_INREG,
ISD::SIGN_EXTEND_VECTOR_INREG,
ISD::ZERO_EXTEND_VECTOR_INREG,
+ ISD::VECREDUCE_MUL,
ISD::SINT_TO_FP,
ISD::UINT_TO_FP,
ISD::FP_TO_SINT,
@@ -34351,6 +34355,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR: return LowerVECREDUCE(Op, Subtarget, DAG, true);
+ case ISD::VECREDUCE_MUL: return LowerVECREDUCE(Op, Subtarget, DAG, false);
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
case ISD::FMINIMUMNUM:
@@ -35153,6 +35158,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
}
return;
}
+ case ISD::VECREDUCE_MUL: {
+ assert(N->getValueType(0) == MVT::i64 && "Unexpected vector reduction");
+ if (SDValue Res = LowerVECREDUCE(SDValue(N, 0), Subtarget, DAG, false))
+ Results.push_back(Res);
+ return;
+ }
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_SMIN:
case ISD::VECREDUCE_UMAX:
@@ -47588,8 +47599,8 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
return SDValue();
ISD::NodeType Opc;
- SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc,
- {ISD::ADD, ISD::MUL, ISD::FADD}, true);
+ SDValue Rdx =
+ DAG.matchBinOpReduction(ExtElt, Opc, {ISD::ADD, ISD::FADD}, true);
if (!Rdx)
return SDValue();
@@ -47606,10 +47617,10 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
unsigned NumElts = VecVT.getVectorNumElements();
unsigned EltSizeInBits = VecVT.getScalarSizeInBits();
- // Extend v4i8/v8i8 vector to v16i8, with undef upper 64-bits.
- auto WidenToV16I8 = [&](SDValue V, bool ZeroExtend) {
+ // ZeroExtend v4i8/v8i8 vector to v16i8, with undef upper 64-bits.
+ auto WidenToV16I8 = [&](SDValue V) {
if (V.getValueType() == MVT::v4i8) {
- if (ZeroExtend && Subtarget.hasSSE41()) {
+ if (Subtarget.hasSSE41()) {
V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4i32,
DAG.getConstant(0, DL, MVT::v4i32),
DAG.getBitcast(MVT::i32, V),
@@ -47617,50 +47628,15 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
return DAG.getBitcast(MVT::v16i8, V);
}
V = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, V,
- ZeroExtend ? DAG.getConstant(0, DL, MVT::v4i8)
- : DAG.getUNDEF(MVT::v4i8));
+ DAG.getConstant(0, DL, MVT::v4i8));
}
return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V,
DAG.getUNDEF(MVT::v8i8));
};
- // vXi8 mul reduction - promote to vXi16 mul reduction.
- if (Opc == ISD::MUL) {
- if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts))
- return SDValue();
- if (VecVT.getSizeInBits() >= 128) {
- EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);
- SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
- SDValue Hi = getUnpackh(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
- Lo = DAG.getBitcast(WideVT, Lo);
- Hi = DAG.getBitcast(WideVT, Hi);
- Rdx = DAG.getNode(Opc, DL, WideVT, Lo, Hi);
- while (Rdx.getValueSizeInBits() > 128) {
- std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
- Rdx = DAG.getNode(Opc, DL, Lo.getValueType(), Lo, Hi);
- }
- } else {
- Rdx = WidenToV16I8(Rdx, false);
- Rdx = getUnpackl(DAG, DL, MVT::v16i8, Rdx, DAG.getUNDEF(MVT::v16i8));
- Rdx = DAG.getBitcast(MVT::v8i16, Rdx);
- }
- if (NumElts >= 8)
- Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
- DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
- {4, 5, 6, 7, -1, -1, -1, -1}));
- Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
- DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
- {2, 3, -1, -1, -1, -1, -1, -1}));
- Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
- DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
- {1, -1, -1, -1, -1, -1, -1, -1}));
- Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
- }
-
// vXi8 add reduction - sub 128-bit vector.
if (VecVT == MVT::v4i8 || VecVT == MVT::v8i8) {
- Rdx = WidenToV16I8(Rdx, true);
+ Rdx = WidenToV16I8(Rdx);
Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
DAG.getConstant(0, DL, MVT::v16i8));
Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
@@ -47706,7 +47682,7 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
EVT ByteVT = VecVT.changeVectorElementType(*DAG.getContext(), MVT::i8);
Rdx = DAG.getNode(ISD::TRUNCATE, DL, ByteVT, Rdx);
if (ByteVT.getSizeInBits() < 128)
- Rdx = WidenToV16I8(Rdx, true);
+ Rdx = WidenToV16I8(Rdx);
}
// Build the PSADBW, split as 128/256/512 bits for SSE/AVX2/AVX512BW.
@@ -47883,7 +47859,7 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget))
return Cmp;
- // Attempt to optimize ADD/FADD/MUL reductions with HADD, promotion etc..
+ // Attempt to optimize ADD/FADD reductions with HADD, promotion etc..
if (SDValue V = combineArithReduction(N, DAG, Subtarget))
return V;
@@ -47972,6 +47948,38 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+static SDValue combineVECREDUCE_MUL(SDNode *N, SelectionDAG &DAG,
+ const X86Subtarget &Subtarget) {
+ SDValue Src = N->getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ unsigned NumElts = SrcVT.getVectorNumElements();
+ SDLoc DL(N);
+
+ // vXi8 mul reduction - promote to vXi16 mul reduction.
+ if (!isPowerOf2_32(NumElts) || SrcVT.getScalarType() != MVT::i8)
+ return SDValue();
+
+ // Early out for v2i8 - no promotion necessary.
+ if (NumElts == 2) {
+ SDValue Hi = DAG.getVectorShuffle(SrcVT, DL, Src, Src, {1, -1});
+ SDValue Rdx = DAG.getNode(ISD::MUL, DL, SrcVT, Src, Hi);
+ return DAG.getExtractVectorElt(DL, N->getValueType(0), Rdx, 0);
+ }
+
+ if (SrcVT.getSizeInBits() >= 128) {
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);
+ SDValue Lo = getUnpackl(DAG, DL, SrcVT, Src, DAG.getUNDEF(SrcVT));
+ SDValue Hi = getUnpackh(DAG, DL, SrcVT, Src, DAG.getUNDEF(SrcVT));
+ Src = DAG.getNode(ISD::MUL, DL, WideVT, DAG.getBitcast(WideVT, Lo),
+ DAG.getBitcast(WideVT, Hi));
+ } else {
+ EVT ExtVT = SrcVT.changeVectorElementType(*DAG.getContext(), MVT::i16);
+ Src = DAG.getNode(ISD::ANY_EXTEND, DL, ExtVT, Src);
+ }
+ Src = DAG.getNode(ISD::VECREDUCE_MUL, DL, MVT::i16, Src);
+ return DAG.getZExtOrTrunc(Src, DL, N->getValueType(0));
+}
+
// Convert (vXiY *ext(vXi1 bitcast(iX))) to extend_in_reg(broadcast(iX)).
// This is more or less the reverse of combineBitcastvxi1.
static SDValue combineToExtendBoolVectorInReg(
@@ -62931,6 +62939,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case X86ISD::VFCMULC:
case X86ISD::VFMULC: return combineFMulcFCMulc(N, DAG, Subtarget);
case ISD::FNEG: return combineFneg(N, DAG, DCI, Subtarget);
+ case ISD::VECREDUCE_MUL: return combineVECREDUCE_MUL(N, DAG, Subtarget);
case ISD::TRUNCATE: return combineTruncate(N, DAG, Subtarget);
case X86ISD::VTRUNC: return combineVTRUNC(N, DAG, DCI);
case X86ISD::VTRUNCS:
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 2cb99d0c2090a..2993173d35edc 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -6851,6 +6851,7 @@ bool X86TTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
switch (II->getIntrinsicID()) {
default:
return true;
+ case Intrinsic::vector_reduce_mul:
case Intrinsic::vector_reduce_smax:
case Intrinsic::vector_reduce_smin:
case Intrinsic::vector_reduce_umax:
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
index 1e14256db3584..ddd94bfd3e4b6 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
@@ -6895,7 +6895,7 @@ define i32 @test_mm512_reduce_mul_epi32(<8 x i64> %__W) nounwind {
; CHECK-LABEL: test_mm512_reduce_mul_epi32:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vpmulld %ymm1, %ymm0, %ymm0
; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-NEXT: vpmulld %xmm1, %xmm0, %xmm0
; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -7000,7 +7000,7 @@ define i32 @test_mm512_mask_reduce_mul_epi32(i16 zeroext %__M, <8 x i64> %__W) n
; X86-NEXT: vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; X86-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
; X86-NEXT: vextracti64x4 $1, %zmm1, %ymm0
-; X86-NEXT: vpmulld %zmm0, %zmm1, %zmm0
+; X86-NEXT: vpmulld %ymm0, %ymm1, %ymm0
; X86-NEXT: vextracti128 $1, %ymm0, %xmm1
; X86-NEXT: vpmulld %xmm1, %xmm0, %xmm0
; X86-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -7017,7 +7017,7 @@ define i32 @test_mm512_mask_reduce_mul_epi32(i16 zeroext %__M, <8 x i64> %__W) n
; X64-NEXT: vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; X64-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
; X64-NEXT: vextracti64x4 $1, %zmm1, %ymm0
-; X64-NEXT: vpmulld %zmm0, %zmm1, %zmm0
+; X64-NEXT: vpmulld %ymm0, %ymm1, %ymm0
; X64-NEXT: vextracti128 $1, %ymm0, %xmm1
; X64-NEXT: vpmulld %xmm1, %xmm0, %xmm0
; X64-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index 103c3329ede5c..b1d5b188b8de2 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -884,7 +884,7 @@ define i64 @test_v8i64(<8 x i64> %a0) nounwind {
; AVX512DQVL-LABEL: test_v8i64:
; AVX512DQVL: # %bb.0:
; AVX512DQVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512DQVL-NEXT: vpmullq %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT: vpmullq %ymm1, %ymm0, %ymm0
; AVX512DQVL-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512DQVL-NEXT: vpmullq %xmm1, %xmm0, %xmm0
; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -903,50 +903,28 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: movl %esp, %ebp
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm4
-; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm3
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm5
-; X86-SSE2-NEXT: psrlq $32, %xmm5
-; X86-SSE2-NEXT: pmuludq %xmm3, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE2-NEXT: psrlq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm2, %xmm6
-; X86-SSE2-NEXT: paddq %xmm5, %xmm6
-; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm5
-; X86-SSE2-NEXT: psllq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm3, %xmm2
-; X86-SSE2-NEXT: paddq %xmm6, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
-; X86-SSE2-NEXT: psrlq $32, %xmm3
-; X86-SSE2-NEXT: pmuludq %xmm5, %xmm3
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm5
+; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm4
+; X86-SSE2-NEXT: psrlq $32, %xmm4
+; X86-SSE2-NEXT: pmuludq %xmm5, %xmm4
; X86-SSE2-NEXT: movdqa %xmm5, %xmm6
; X86-SSE2-NEXT: psrlq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm0, %xmm6
-; X86-SSE2-NEXT: paddq %xmm3, %xmm6
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT: pmuludq %xmm3, %xmm6
+; X86-SSE2-NEXT: paddq %xmm4, %xmm6
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm4
; X86-SSE2-NEXT: psllq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm5, %xmm0
-; X86-SSE2-NEXT: paddq %xmm6, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm3, %xmm5
+; X86-SSE2-NEXT: pmuludq %xmm5, %xmm3
+; X86-SSE2-NEXT: paddq %xmm6, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm5
; X86-SSE2-NEXT: psrlq $32, %xmm5
; X86-SSE2-NEXT: pmuludq %xmm4, %xmm5
; X86-SSE2-NEXT: movdqa %xmm4, %xmm6
; X86-SSE2-NEXT: psrlq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm3, %xmm6
-; X86-SSE2-NEXT: paddq %xmm5, %xmm6
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm5
-; X86-SSE2-NEXT: psllq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm4, %xmm3
-; X86-SSE2-NEXT: paddq %xmm6, %xmm3
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
-; X86-SSE2-NEXT: psrlq $32, %xmm4
-; X86-SSE2-NEXT: pmuludq %xmm5, %xmm4
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm6
-; X86-SSE2-NEXT: psrlq $32, %xmm6
; X86-SSE2-NEXT: pmuludq %xmm1, %xmm6
-; X86-SSE2-NEXT: paddq %xmm4, %xmm6
+; X86-SSE2-NEXT: paddq %xmm5, %xmm6
; X86-SSE2-NEXT: psllq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm5, %xmm1
+; X86-SSE2-NEXT: pmuludq %xmm4, %xmm1
; X86-SSE2-NEXT: paddq %xmm6, %xmm1
; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
; X86-SSE2-NEXT: psrlq $32, %xmm4
@@ -955,9 +933,31 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: psrlq $32, %xmm5
; X86-SSE2-NEXT: pmuludq %xmm1, %xmm5
; X86-SSE2-NEXT: paddq %xmm4, %xmm5
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm4
; X86-SSE2-NEXT: psllq $32, %xmm5
; X86-SSE2-NEXT: pmuludq %xmm3, %xmm1
; X86-SSE2-NEXT: paddq %xmm5, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
+; X86-SSE2-NEXT: psrlq $32, %xmm3
+; X86-SSE2-NEXT: pmuludq %xmm4, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm5
+; X86-SSE2-NEXT: psrlq $32, %xmm5
+; X86-SSE2-NEXT: pmuludq %xmm2, %xmm5
+; X86-SSE2-NEXT: paddq %xmm3, %xmm5
+; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm3
+; X86-SSE2-NEXT: psllq $32, %xmm5
+; X86-SSE2-NEXT: pmuludq %xmm4, %xmm2
+; X86-SSE2-NEXT: paddq %xmm5, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm4
+; X86-SSE2-NEXT: psrlq $32, %xmm4
+; X86-SSE2-NEXT: pmuludq %xmm3, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm5
+; X86-SSE2-NEXT: psrlq $32, %xmm5
+; X86-SSE2-NEXT: pmuludq %xmm0, %xmm5
+; X86-SSE2-NEXT: paddq %xmm4, %xmm5
+; X86-SSE2-NEXT: psllq $32, %xmm5
+; X86-SSE2-NEXT: pmuludq %xmm3, %xmm0
+; X86-SSE2-NEXT: paddq %xmm5, %xmm0
; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
; X86-SSE2-NEXT: psrlq $32, %xmm3
; X86-SSE2-NEXT: pmuludq %xmm2, %xmm3
@@ -998,56 +998,56 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
;
; X64-SSE-LABEL: test_v16i64:
; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: movdqa %xmm2, %xmm8
+; X64-SSE-NEXT: movdqa %xmm3, %xmm8
; X64-SSE-NEXT: psrlq $32, %xmm8
-; X64-SSE-NEXT: pmuludq %xmm6, %xmm8
-; X64-SSE-NEXT: movdqa %xmm6, %xmm9
+; X64-SSE-NEXT: pmuludq %xmm7, %xmm8
+; X64-SSE-NEXT: movdqa %xmm7, %xmm9
; X64-SSE-NEXT: psrlq $32, %xmm9
-; X64-SSE-NEXT: pmuludq %xmm2, %xmm9
+; X64-SSE-NEXT: pmuludq %xmm3, %xmm9
; X64-SSE-NEXT: paddq %xmm8, %xmm9
; X64-SSE-NEXT: psllq $32, %xmm9
-; X64-SSE-NEXT: pmuludq %xmm6, %xmm2
-; X64-SSE-NEXT: paddq %xmm9, %xmm2
-; X64-SSE-NEXT: movdqa %xmm0, %xmm6
-; X64-SSE-NEXT: psrlq $32, %xmm6
-; X64-SSE-NEXT: pmuludq %xmm4, %xmm6
-; X64-SSE-NEXT: movdqa %xmm4, %xmm8
+; X64-SSE-NEXT: pmuludq %xmm7, %xmm3
+; X64-SSE-NEXT: paddq %xmm9, %xmm3
+; X64-SSE-NEXT: movdqa %xmm1, %xmm7
+; X64-SSE-NEXT: psrlq $32, %xmm7
+; X64-SSE-NEXT: pmuludq %xmm5, %xmm7
+; X64-SSE-NEXT: movdqa %xmm5, %xmm8
; X64-SSE-NEXT: psrlq $32, %xmm8
-; X64-SSE-NEXT: pmuludq %xmm0, %xmm8
-; X64-SSE-NEXT: paddq %xmm6, %xmm8
+; X64-SSE-NEXT: pmuludq %xmm1, %xmm8
+; X64-SSE-NEXT: paddq %xmm7, %xmm8
; X64-SSE-NEXT: psllq $32, %xmm8
-; X64-SSE-NEXT: pmuludq %xmm4, %xmm0
-; X64-SSE-NEXT: paddq %xmm8, %xmm0
-; X64-SSE-NEXT: movdqa %xmm3, %xmm4
-; X64-SSE-NEXT: psrlq $32, %xmm4
-; X64-SSE-NEXT: pmuludq %xmm7, %xmm4
-; X64-SSE-NEXT: movdqa %xmm7, %xmm6
-; X64-SSE-NEXT: psrlq $32, %xmm6
-; X64-SSE-NEXT: pmuludq %xmm3, %xmm6
-; X64-SSE-NEXT: paddq %xmm4, %xmm6
-; X64-SSE-NEXT: psllq $32, %xmm6
-; X64-SSE-NEXT: pmuludq %xmm7, %xmm3
-; X64-SSE-NEXT: paddq %xmm6, %xmm3
-; X64-SSE-NEXT: movdqa %xmm1, %xmm4
-; X64-SSE-NEXT: psrlq $32, %xmm4
-; X64-SSE-NEXT: pmuludq %xmm5, %xmm4
-; X64-SSE-NEXT: movdqa %xmm5, %xmm6
-; X64-SSE-NEXT: psrlq $32, %xmm6
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm6
-; X64-SSE-NEXT: paddq %xmm4, %xmm6
-; X64-SSE-NEXT: psllq $32, %xmm6
; X64-SSE-NEXT: pmuludq %xmm5, %xmm1
-; X64-SSE-NEXT: paddq %xmm6, %xmm1
-; X64-SSE-NEXT: movdqa %xmm1, %xmm4
-; X64-SSE-NEXT: psrlq $32, %xmm4
-; X64-SSE-NEXT: pmuludq %xmm3, %xmm4
-; X64-SSE-NEXT: movdqa %xmm3, %xmm5
+; X64-SSE-NEXT: paddq %xmm8, %xmm1
+; X64-SSE-NEXT: movdqa %xmm1, %xmm5
; X64-SSE-NEXT: psrlq $32, %xmm5
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm5
-; X64-SSE-NEXT: paddq %xmm4, %xmm5
-; X64-SSE-NEXT: psllq $32, %xmm5
+; X64-SSE-NEXT: pmuludq %xmm3, %xmm5
+; X64-SSE-NEXT: movdqa %xmm3, %xmm7
+; X64-SSE-NEXT: psrlq $32, %xmm7
+; X64-SSE-NEXT: pmuludq %xmm1, %xmm7
+; X64-SSE-NEXT: paddq %xmm5, %xmm7
+; X64-SSE-NEXT: psllq $32, %xmm7
; X64-SSE-NEXT: pmuludq %xmm3, %xmm1
-; X64-SSE-NEXT: paddq %xmm5, %xmm1
+; X64-SSE-NEXT: paddq %xmm7, %xmm1
+; X64-SSE-NEXT: movdqa %xmm2, %xmm3
+; X64-SSE-NEXT: psrlq $32, %xmm3
+; X64-SSE-NEXT: pmuludq %xmm6, %xmm3
+; X64-SSE-NEXT: movdqa %xmm6, %xmm5
+; X64-SSE-NEXT: psrlq $32, %xmm5
+; X64-SSE-NEXT: pmuludq %xmm2, %xmm5
+; X64-SSE-NEXT: paddq %xmm3, %xmm5
+; X64-SSE-NEXT: psllq $32, %xmm5
+; X64-SSE-NEXT: pmuludq %xmm6, %xmm2
+; X64-SSE-NEXT: paddq %xmm5, %xmm2
+; X64-SSE-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE-NEXT: psrlq $32, %xmm3
+; X64-SSE-NEXT: pmuludq %xmm4, %xmm3
+; X64-SSE-NEXT: movdqa %xmm4, %xmm5
+; X64-SSE-NEXT: psrlq $32, %xmm5
+; X64-SSE-NEXT: pmuludq %xmm0, %xmm5
+; X64-SSE-NEXT: paddq %xmm3, %xmm5
+; X64-SSE-NEXT: psllq $32, %xmm5
+; X64-SSE-NEXT: pmuludq %xmm4, %xmm0
+; X64-SSE-NEXT: paddq %xmm5, %xmm0
; X64-SSE-NEXT: movdqa %xmm0, %xmm3
; X64-SSE-NEXT: psrlq $32, %xmm3
; X64-SSE-NEXT: pmuludq %xmm2, %xmm3
@@ -1088,50 +1088,28 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE4-NEXT: movl %esp, %ebp
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
-; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm4
-; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm3
-; X86-SSE4-NEXT: movdqa %xmm2, %xmm5
-; X86-SSE4-NEXT: psrlq $32, %xmm5
-; X86-SSE4-NEXT: pmuludq %xmm3, %xmm5
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE4-NEXT: psrlq $32, %xmm6
-; X86-SSE4-NEXT: pmuludq %xmm2, %xmm6
-; X86-SSE4-NEXT: paddq %xmm5, %xmm6
-; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm5
-; X86-SSE4-NEXT: psllq $32, %xmm6
-; X86-SSE4-NEXT: pmuludq %xmm3, %xmm2
-; X86-SSE4-NEXT: paddq %xmm6, %xmm2
-; X86-SSE4-NEXT: movdqa %xmm0, %xmm3
-; X86-SSE4-NEXT: psrlq $32, %xmm3
-; X86-SSE4-NEXT: pmuludq %xmm5, %xmm3
+; X86-SSE4-NEXT: movdqa 72(%ebp), %xmm5
+; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm3
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm4
+; X86-SSE4-NEXT: psrlq $32, %xmm4
+; X86-SSE4-NEXT: pmuludq %xmm5, %xmm4
; X86-SSE4-NEXT: movdqa %xmm5, %xmm6
; X86-SSE4-NEXT: psrlq $32, %xmm6
-; X86-SSE4-NEXT: pmuludq %xmm0, %xmm6
-; X86-SSE4-NEXT: paddq %xmm3, %xmm6
-; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm3
+; X86-SSE4-NEXT: pmuludq %xmm3, %xmm6
+; X86-SSE4-NEXT: paddq %xmm4, %xmm6
+; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm4
; X86-SSE4-NEXT: psllq $32, %xmm6
-; X86-SSE4-NEXT: pmuludq %xmm5, %xmm0
-; X86-SSE4-NEXT: paddq %xmm6, %xmm0
-; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
+; X86-SSE4-NEXT: pmuludq %xmm5, %xmm3
+; X86-SSE4-NEXT: paddq %xmm6, %xmm3
+; X86-SSE4-NEXT: movdqa %xmm1, %xmm5
; X86-SSE4-NEXT: psrlq $32, %xmm5
; X86-SSE4-NEXT: pmuludq %xmm4, %xmm5
; X86-SSE4-NEXT: movdqa %xmm4, %xmm6
; X86-SSE4-NEXT: psrlq $32, %xmm6
-; X86-SSE4-NEXT: pmuludq %xmm3, %xmm6
-; X86-SSE4-NEXT: paddq %xmm5, %xmm6
-; X86-SSE4-NEXT: movdqa 40(%ebp), %xmm5
-; X86-SSE4-NEXT: psllq $32, %xmm6
-; X86-SSE4-NEXT: pmuludq %xmm4, %xmm3
-; X86-SSE4-NEXT: paddq %xmm6, %xmm3
-; X86-SSE4-NEXT: movdqa %xmm1, %xmm4
-; X86-SSE4-NEXT: psrlq $32, %xmm4
-; X86-SSE4-NEXT: pmuludq %xmm5, %xmm4
-; X86-SSE4-NEXT: movdqa %xmm5, %xmm6
-; X86-SSE4-NEXT: psrlq $32, %xmm6
; X86-SSE4-NEXT: pmuludq %xmm1, %xmm6
-; X86-SSE4-NEXT: paddq %xmm4, %xmm6
+; X86-SSE4-NEXT: paddq %xmm5, %xmm6
; X86-SSE4-NEXT: psllq $32, %xmm6
-; X86-SSE4-NEXT: pmuludq %xmm5, %xmm1
+; X86-SSE4-NEXT: pmuludq %xmm4, %xmm1
; X86-SSE4-NEXT: paddq %xmm6, %xmm1
; X86-SSE4-NEXT: movdqa %xmm1, %xmm4
; X86-SSE4-NEXT: psrlq $32, %xmm4
@@ -1140,9 +1118,31 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE4-NEXT: psrlq $32, %xmm5
; X86-SSE4-NEXT: pmuludq %xmm1, %xmm5
; X86-SSE4-NEXT: paddq %xmm4, %xmm5
+; X86-SSE4-NEXT: movdqa 56(%ebp), %xmm4
; X86-SSE4-NEXT: psllq $32, %xmm5
; X86-SSE4-NEXT: pmuludq %xmm3, %xmm1
; X86-SSE4-NEXT: paddq %xmm5, %xmm1
+; X86-SSE4-NEXT: movdqa %xmm2, %xmm3
+; X86-SSE4-NEXT: psrlq $32, %xmm3
+; X86-SSE4-NEXT: pmuludq %xmm4, %xmm3
+; X86-SSE4-NEXT: movdqa %xmm4, %xmm5
+; X86-SSE4-NEXT: psrlq $32, %xmm5
+; X86-SSE4-NEXT: pmuludq %xmm2, %xmm5
+; X86-SSE4-NEXT: paddq %xmm3, %xmm5
+; X86-SSE4-NEXT: movdqa 24(%ebp), %xmm3
+; X86-SSE4-NEXT: psllq $32, %xmm5
+; X86-SSE4-NEXT: pmuludq %xmm4, %xmm2
+; X86-SSE4-NEXT: paddq %xmm5, %xmm2
+; X86-SSE4-NEXT: movdqa %xmm0, %xmm4
+; X86-SSE4-NEXT: psrlq $32, %xmm4
+; X86-SSE4-NEXT: pmuludq %xmm3, %xmm4
+; X86-SSE4-NEXT: movdqa %xmm3, %xmm5
+; X86-SSE4-NEXT: psrlq $32, %xmm5
+; X86-SSE4-NEXT: pmuludq %xmm0, %xmm5
+; X86-SSE4-NEXT: paddq %xmm4, %xmm5
+; X86-SSE4-NEXT: psllq $32, %xmm5
+; X86-SSE4-NEXT: pmuludq %xmm3, %xmm0
+; X86-SSE4-NEXT: paddq %xmm5, %xmm0
; X86-SSE4-NEXT: movdqa %xmm0, %xmm3
; X86-SSE4-NEXT: psrlq $32, %xmm3
; X86-SSE4-NEXT: pmuludq %xmm2, %xmm3
@@ -1186,76 +1186,76 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-AVX1-NEXT: movl %esp, %ebp
; X86-AVX1-NEXT: andl $-32, %esp
; X86-AVX1-NEXT: subl $32, %esp
-; X86-AVX1-NEXT: vmovdqa 8(%ebp), %xmm3
-; X86-AVX1-NEXT: vmovdqa 24(%ebp), %xmm5
+; X86-AVX1-NEXT: vmovdqa 24(%ebp), %xmm3
; X86-AVX1-NEXT: vpsrlq $32, %xmm3, %xmm4
-; X86-AVX1-NEXT: vpmuludq %xmm4, %xmm1, %xmm4
-; X86-AVX1-NEXT: vpsrlq $32, %xmm1, %xmm6
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X86-AVX1-NEXT: vpmuludq %xmm4, %xmm5, %xmm4
+; X86-AVX1-NEXT: vpsrlq $32, %xmm5, %xmm6
; X86-AVX1-NEXT: vpmuludq %xmm3, %xmm6, %xmm6
; X86-AVX1-NEXT: vpaddq %xmm6, %xmm4, %xmm4
; X86-AVX1-NEXT: vpsllq $32, %xmm4, %xmm4
-; X86-AVX1-NEXT: vpmuludq %xmm3, %xmm1, %xmm3
+; X86-AVX1-NEXT: vpmuludq %xmm3, %xmm5, %xmm3
; X86-AVX1-NEXT: vpaddq %xmm4, %xmm3, %xmm3
-; X86-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm4
-; X86-AVX1-NEXT: vpmuludq %xmm2, %xmm4, %xmm4
-; X86-AVX1-NEXT: vpsrlq $32, %xmm2, %xmm6
-; X86-AVX1-NEXT: vpmuludq %xmm6, %xmm0, %xmm6
-; X86-AVX1-NEXT: vpaddq %xmm4, %xmm6, %xmm4
-; X86-AVX1-NEXT: vpsllq $32, %xmm4, %xmm4
-; X86-AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm6
-; X86-AVX1-NEXT: vpaddq %xmm4, %xmm6, %xmm4
+; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm4
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm5
; X86-AVX1-NEXT: vpsrlq $32, %xmm5, %xmm6
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-NEXT: vpmuludq %xmm6, %xmm1, %xmm6
-; X86-AVX1-NEXT: vpsrlq $32, %xmm1, %xmm7
-; X86-AVX1-NEXT: vpmuludq %xmm5, %xmm7, %xmm7
-; X86-AVX1-NEXT: vpaddq %xmm7, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpmuludq %xmm4, %xmm6, %xmm6
+; X86-AVX1-NEXT: vpsrlq $32, %xmm4, %xmm7
+; X86-AVX1-NEXT: vpmuludq %xmm7, %xmm5, %xmm7
+; X86-AVX1-NEXT: vpaddq %xmm6, %xmm7, %xmm6
; X86-AVX1-NEXT: vpsllq $32, %xmm6, %xmm6
-; X86-AVX1-NEXT: vpmuludq %xmm5, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpaddq %xmm6, %xmm1, %xmm1
-; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
-; X86-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm5
-; X86-AVX1-NEXT: vpmuludq %xmm2, %xmm5, %xmm5
-; X86-AVX1-NEXT: vpsrlq $32, %xmm2, %xmm6
-; X86-AVX1-NEXT: vpmuludq %xmm6, %xmm0, %xmm6
+; X86-AVX1-NEXT: vpmuludq %xmm4, %xmm5, %xmm4
+; X86-AVX1-NEXT: vpaddq %xmm6, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpsrlq $32, %xmm4, %xmm5
+; X86-AVX1-NEXT: vpmuludq %xmm3, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpsrlq $32, %xmm3, %xmm6
+; X86-AVX1-NEXT: vpmuludq %xmm6, %xmm4, %xmm6
; X86-AVX1-NEXT: vpaddq %xmm5, %xmm6, %xmm5
+; X86-AVX1-NEXT: vmovdqa 8(%ebp), %xmm6
; X86-AVX1-NEXT: vpsllq $32, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpmuludq %xmm3, %xmm4, %xmm3
+; X86-AVX1-NEXT: vpaddq %xmm5, %xmm3, %xmm3
+; X86-AVX1-NEXT: vpsrlq $32, %xmm6, %xmm4
+; X86-AVX1-NEXT: vpmuludq %xmm4, %xmm1, %xmm4
+; X86-AVX1-NEXT: vpsrlq $32, %xmm1, %xmm5
+; X86-AVX1-NEXT: vpmuludq %xmm6, %xmm5, %xmm5
+; X86-AVX1-NEXT: vpaddq %xmm5, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpsllq $32, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpmuludq %xmm6, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpaddq %xmm4, %xmm1, %xmm1
+; X86-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm4
+; X86-AVX1-NEXT: vpmuludq %xmm2, %xmm4, %xmm4
+; X86-AVX1-NEXT: vpsrlq $32, %xmm2, %xmm5
+; X86-AVX1-NEXT: vpmuludq %xmm5, %xmm0, %xmm5
+; X86-AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
+; X86-AVX1-NEXT: vpsllq $32, %xmm4, %xmm4
; X86-AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm5, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddq %xmm4, %xmm0, %xmm0
; X86-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2
; X86-AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpsrlq $32, %xmm1, %xmm5
-; X86-AVX1-NEXT: vpmuludq %xmm5, %xmm0, %xmm5
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm5, %xmm2
+; X86-AVX1-NEXT: vpsrlq $32, %xmm1, %xmm4
+; X86-AVX1-NEXT: vpmuludq %xmm4, %xmm0, %xmm4
+; X86-AVX1-NEXT: vpaddq %xmm2, %xmm4, %xmm2
; X86-AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
; X86-AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpsrlq $32, %xmm4, %xmm1
+; X86-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm1
; X86-AVX1-NEXT: vpmuludq %xmm3, %xmm1, %xmm1
; X86-AVX1-NEXT: vpsrlq $32, %xmm3, %xmm2
-; X86-AVX1-NEXT: vpmuludq %xmm2, %xmm4, %xmm2
+; X86-AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
; X86-AVX1-NEXT: vpaddq %xmm1, %xmm2, %xmm1
; X86-AVX1-NEXT: vpsllq $32, %xmm1, %xmm1
-; X86-AVX1-NEXT: vpmuludq %xmm3, %xmm4, %xmm2
-; X86-AVX1-NEXT: vpaddq %xmm1, %xmm2, %xmm1
-; X86-AVX1-NEXT: vpsrlq $32, %xmm1, %xmm2
-; X86-AVX1-NEXT: vpmuludq %xmm0, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm3
-; X86-AVX1-NEXT: vpmuludq %xmm3, %xmm1, %xmm3
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X86-AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
-; X86-AVX1-NEXT: vpmuludq %xmm0, %xmm1, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2
; X86-AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
; X86-AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
; X86-AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
; X86-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X86-AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
; X86-AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X86-AVX1-NEXT: vpsllq $32, %xmm2, %xmm1
+; X86-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vmovd %xmm0, %eax
; X86-AVX1-NEXT: vpextrd $1, %xmm0, %edx
; X86-AVX1-NEXT: movl %ebp, %esp
@@ -1265,39 +1265,47 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
;
; X64-AVX1-LABEL: test_v16i64:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vpsrlq $32, %xmm1, %xmm4
-; X64-AVX1-NEXT: vpmuludq %xmm3, %xmm4, %xmm4
-; X64-AVX1-NEXT: vpsrlq $32, %xmm3, %xmm5
-; X64-AVX1-NEXT: vpmuludq %xmm5, %xmm1, %xmm5
-; X64-AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
-; X64-AVX1-NEXT: vpsllq $32, %xmm4, %xmm4
-; X64-AVX1-NEXT: vpmuludq %xmm3, %xmm1, %xmm5
-; X64-AVX1-NEXT: vpaddq %xmm4, %xmm5, %xmm4
-; X64-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm5
-; X64-AVX1-NEXT: vpmuludq %xmm2, %xmm5, %xmm5
-; X64-AVX1-NEXT: vpsrlq $32, %xmm2, %xmm6
-; X64-AVX1-NEXT: vpmuludq %xmm6, %xmm0, %xmm6
-; X64-AVX1-NEXT: vpaddq %xmm5, %xmm6, %xmm5
-; X64-AVX1-NEXT: vpsllq $32, %xmm5, %xmm5
-; X64-AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm6
-; X64-AVX1-NEXT: vpaddq %xmm5, %xmm6, %xmm5
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X64-AVX1-NEXT: vpsrlq $32, %xmm1, %xmm6
-; X64-AVX1-NEXT: vpmuludq %xmm3, %xmm6, %xmm6
-; X64-AVX1-NEXT: vpsrlq $32, %xmm3, %xmm7
-; X64-AVX1-NEXT: vpmuludq %xmm7, %xmm1, %xmm7
+; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-NEXT: vpsrlq $32, %xmm5, %xmm6
+; X64-AVX1-NEXT: vpmuludq %xmm4, %xmm6, %xmm6
+; X64-AVX1-NEXT: vpsrlq $32, %xmm4, %xmm7
+; X64-AVX1-NEXT: vpmuludq %xmm7, %xmm5, %xmm7
+; X64-AVX1-NEXT: vpaddq %xmm6, %xmm7, %xmm6
+; X64-AVX1-NEXT: vpsllq $32, %xmm6, %xmm6
+; X64-AVX1-NEXT: vpmuludq %xmm4, %xmm5, %xmm4
+; X64-AVX1-NEXT: vpaddq %xmm6, %xmm4, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-NEXT: vpsrlq $32, %xmm6, %xmm7
+; X64-AVX1-NEXT: vpmuludq %xmm5, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpsrlq $32, %xmm5, %xmm8
+; X64-AVX1-NEXT: vpmuludq %xmm6, %xmm8, %xmm8
+; X64-AVX1-NEXT: vpaddq %xmm7, %xmm8, %xmm7
+; X64-AVX1-NEXT: vpsllq $32, %xmm7, %xmm7
+; X64-AVX1-NEXT: vpmuludq %xmm5, %xmm6, %xmm5
+; X64-AVX1-NEXT: vpaddq %xmm7, %xmm5, %xmm5
+; X64-AVX1-NEXT: vpsrlq $32, %xmm5, %xmm6
+; X64-AVX1-NEXT: vpmuludq %xmm4, %xmm6, %xmm6
+; X64-AVX1-NEXT: vpsrlq $32, %xmm4, %xmm7
+; X64-AVX1-NEXT: vpmuludq %xmm7, %xmm5, %xmm7
; X64-AVX1-NEXT: vpaddq %xmm6, %xmm7, %xmm6
; X64-AVX1-NEXT: vpsllq $32, %xmm6, %xmm6
+; X64-AVX1-NEXT: vpmuludq %xmm4, %xmm5, %xmm4
+; X64-AVX1-NEXT: vpaddq %xmm6, %xmm4, %xmm4
+; X64-AVX1-NEXT: vpsrlq $32, %xmm1, %xmm5
+; X64-AVX1-NEXT: vpmuludq %xmm3, %xmm5, %xmm5
+; X64-AVX1-NEXT: vpsrlq $32, %xmm3, %xmm6
+; X64-AVX1-NEXT: vpmuludq %xmm6, %xmm1, %xmm6
+; X64-AVX1-NEXT: vpaddq %xmm5, %xmm6, %xmm5
+; X64-AVX1-NEXT: vpsllq $32, %xmm5, %xmm5
; X64-AVX1-NEXT: vpmuludq %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpaddq %xmm6, %xmm1, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X64-AVX1-NEXT: vpaddq %xmm5, %xmm1, %xmm1
; X64-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm3
; X64-AVX1-NEXT: vpmuludq %xmm2, %xmm3, %xmm3
-; X64-AVX1-NEXT: vpsrlq $32, %xmm2, %xmm6
-; X64-AVX1-NEXT: vpmuludq %xmm6, %xmm0, %xmm6
-; X64-AVX1-NEXT: vpaddq %xmm3, %xmm6, %xmm3
+; X64-AVX1-NEXT: vpsrlq $32, %xmm2, %xmm5
+; X64-AVX1-NEXT: vpmuludq %xmm5, %xmm0, %xmm5
+; X64-AVX1-NEXT: vpaddq %xmm3, %xmm5, %xmm3
; X64-AVX1-NEXT: vpsllq $32, %xmm3, %xmm3
; X64-AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm0
; X64-AVX1-NEXT: vpaddq %xmm3, %xmm0, %xmm0
@@ -1309,22 +1317,14 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X64-AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
; X64-AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpsrlq $32, %xmm5, %xmm1
+; X64-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm1
; X64-AVX1-NEXT: vpmuludq %xmm4, %xmm1, %xmm1
; X64-AVX1-NEXT: vpsrlq $32, %xmm4, %xmm2
-; X64-AVX1-NEXT: vpmuludq %xmm2, %xmm5, %xmm2
+; X64-AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm2
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm2, %xmm1
; X64-AVX1-NEXT: vpsllq $32, %xmm1, %xmm1
-; X64-AVX1-NEXT: vpmuludq %xmm4, %xmm5, %xmm2
-; X64-AVX1-NEXT: vpaddq %xmm1, %xmm2, %xmm1
-; X64-AVX1-NEXT: vpsrlq $32, %xmm1, %xmm2
-; X64-AVX1-NEXT: vpmuludq %xmm0, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm3
-; X64-AVX1-NEXT: vpmuludq %xmm3, %xmm1, %xmm3
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X64-AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpmuludq %xmm0, %xmm1, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpmuludq %xmm4, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2
; X64-AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
@@ -1582,7 +1582,7 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; AVX512DQVL: # %bb.0:
; AVX512DQVL-NEXT: vpmullq %zmm1, %zmm0, %zmm0
; AVX512DQVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512DQVL-NEXT: vpmullq %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT: vpmullq %ymm1, %ymm0, %ymm0
; AVX512DQVL-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512DQVL-NEXT: vpmullq %xmm1, %xmm0, %xmm0
; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1626,13 +1626,13 @@ define i32 @test_v2i32(<2 x i32> %a0) nounwind {
define i32 @test_v4i32(<4 x i32> %a0) nounwind {
; SSE2-LABEL: test_v4i32:
; SSE2: # %bb.0:
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
+; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
+; SSE2-NEXT: pmuludq %xmm1, %xmm2
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[3,3,3,3]
-; SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; SSE2-NEXT: pmuludq %xmm2, %xmm3
; SSE2-NEXT: pmuludq %xmm0, %xmm1
-; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm3[0,0,0,0]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,0,0]
; SSE2-NEXT: pmuludq %xmm1, %xmm0
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: ret{{[l|q]}}
@@ -1865,7 +1865,7 @@ define i32 @test_v16i32(<16 x i32> %a0) nounwind {
; AVX512-LABEL: test_v16i32:
; AVX512: # %bb.0:
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmulld %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -1886,67 +1886,67 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X86-SSE2-NEXT: movl %esp, %ebp
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm5
-; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm7
-; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm4
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[1,1,3,3]
-; X86-SSE2-NEXT: pmuludq %xmm4, %xmm2
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm2[0,2,2,3]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[1,1,3,3]
-; X86-SSE2-NEXT: pmuludq %xmm3, %xmm2
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm2[0,2,2,3]
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[1,1,3,3]
-; X86-SSE2-NEXT: pmuludq %xmm7, %xmm0
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm7[1,1,3,3]
-; X86-SSE2-NEXT: pmuludq %xmm3, %xmm0
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm0[0,2,2,3]
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm3[0],xmm4[1],xmm3[1]
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE2-NEXT: pmuludq %xmm6, %xmm4
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; X86-SSE2-NEXT: pmuludq %xmm5, %xmm3
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[0,2,2,3]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; X86-SSE2-NEXT: pmuludq %xmm7, %xmm5
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm5[0,2,2,3]
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm3[0],xmm6[1],xmm3[1]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,3,3]
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm7
-; X86-SSE2-NEXT: pmuludq %xmm7, %xmm1
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm3
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm4
+; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm6
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
+; X86-SSE2-NEXT: pmuludq %xmm4, %xmm6
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; X86-SSE2-NEXT: pmuludq %xmm4, %xmm5
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,2,2,3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; X86-SSE2-NEXT: pmuludq %xmm3, %xmm1
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; X86-SSE2-NEXT: pmuludq %xmm3, %xmm7
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
+; X86-SSE2-NEXT: pmuludq %xmm4, %xmm7
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm7[0,2,2,3]
; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm4
; X86-SSE2-NEXT: pmuludq %xmm6, %xmm1
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm6
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,2,2]
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,0,2,2]
; X86-SSE2-NEXT: pmuludq %xmm5, %xmm1
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3]
; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
+; X86-SSE2-NEXT: pmuludq %xmm6, %xmm2
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,2,2,3]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
+; X86-SSE2-NEXT: pmuludq %xmm7, %xmm2
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm2[0,2,2,3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm0[1,1,3,3]
+; X86-SSE2-NEXT: pmuludq %xmm4, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; X86-SSE2-NEXT: pmuludq %xmm6, %xmm4
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm4[0,2,2,3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm6[0],xmm0[1],xmm6[1]
+; X86-SSE2-NEXT: pmuludq %xmm5, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
-; X86-SSE2-NEXT: pmuludq %xmm2, %xmm0
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm4 = xmm4[0],xmm2[0],xmm4[1],xmm2[1]
-; X86-SSE2-NEXT: pmuludq %xmm3, %xmm4
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,2,2]
+; X86-SSE2-NEXT: pmuludq %xmm2, %xmm4
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X86-SSE2-NEXT: pmuludq %xmm3, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
-; X86-SSE2-NEXT: pmuludq %xmm1, %xmm0
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[0,2,2,3]
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
-; X86-SSE2-NEXT: pmuludq %xmm2, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,2,2,2]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
-; X86-SSE2-NEXT: pmuludq %xmm2, %xmm0
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm0[0],xmm1[1],xmm0[1]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,0,2,2]
+; X86-SSE2-NEXT: pmuludq %xmm1, %xmm2
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X86-SSE2-NEXT: pmuludq %xmm0, %xmm1
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
+; X86-SSE2-NEXT: pmuludq %xmm0, %xmm2
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,0,0]
; X86-SSE2-NEXT: pmuludq %xmm1, %xmm0
; X86-SSE2-NEXT: movd %xmm0, %eax
; X86-SSE2-NEXT: movl %ebp, %esp
@@ -1955,63 +1955,63 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
;
; X64-SSE2-LABEL: test_v32i32:
; X64-SSE2: # %bb.0:
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm2[1,1,3,3]
-; X64-SSE2-NEXT: pmuludq %xmm6, %xmm2
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm9 = xmm2[0,2,2,3]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm6[1,1,3,3]
-; X64-SSE2-NEXT: pmuludq %xmm8, %xmm2
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm2[0,2,2,3]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm6[0],xmm9[1],xmm6[1]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm0[1,1,3,3]
-; X64-SSE2-NEXT: pmuludq %xmm4, %xmm0
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm0[0,2,2,3]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,3,3]
-; X64-SSE2-NEXT: pmuludq %xmm8, %xmm0
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm0[0,2,2,3]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
-; X64-SSE2-NEXT: pmuludq %xmm9, %xmm6
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm3[1,1,3,3]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm3[1,1,3,3]
; X64-SSE2-NEXT: pmuludq %xmm7, %xmm3
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm3[0,2,2,3]
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[1,1,3,3]
-; X64-SSE2-NEXT: pmuludq %xmm4, %xmm7
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,2,2,3]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; X64-SSE2-NEXT: pmuludq %xmm8, %xmm7
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm7[0,2,2,3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm8[0],xmm3[1],xmm8[1]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm1[1,1,3,3]
; X64-SSE2-NEXT: pmuludq %xmm5, %xmm1
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[1,1,3,3]
-; X64-SSE2-NEXT: pmuludq %xmm4, %xmm5
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,2,2,3]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
+; X64-SSE2-NEXT: pmuludq %xmm8, %xmm5
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm5[0,2,2,3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1]
; X64-SSE2-NEXT: pmuludq %xmm3, %xmm1
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm7[0,0,2,2]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,0,2,2]
-; X64-SSE2-NEXT: pmuludq %xmm3, %xmm4
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm4[0,2,2,3]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm6[0,2,2,3]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,0,2,2]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm5[0,0,2,2]
+; X64-SSE2-NEXT: pmuludq %xmm7, %xmm1
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
+; X64-SSE2-NEXT: pmuludq %xmm6, %xmm2
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; X64-SSE2-NEXT: pmuludq %xmm5, %xmm6
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[0,2,2,3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; X64-SSE2-NEXT: pmuludq %xmm4, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
+; X64-SSE2-NEXT: pmuludq %xmm5, %xmm4
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm4[0,2,2,3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm5[0],xmm0[1],xmm5[1]
; X64-SSE2-NEXT: pmuludq %xmm2, %xmm0
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm2[0],xmm3[1],xmm2[1]
-; X64-SSE2-NEXT: pmuludq %xmm1, %xmm3
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,2,2,3]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm6[0,0,2,2]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,0,2,2]
+; X64-SSE2-NEXT: pmuludq %xmm2, %xmm4
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; X64-SSE2-NEXT: pmuludq %xmm3, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2]
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,0,2,2]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
-; X64-SSE2-NEXT: pmuludq %xmm2, %xmm0
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[0,2,2,3]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[2,3,2,3]
; X64-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,2,2,2]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,2,2]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; X64-SSE2-NEXT: pmuludq %xmm0, %xmm1
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
+; X64-SSE2-NEXT: pmuludq %xmm0, %xmm2
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,0,0]
; X64-SSE2-NEXT: pmuludq %xmm1, %xmm0
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm0[0],xmm2[1],xmm0[1]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,0,0,0]
-; X64-SSE2-NEXT: pmuludq %xmm2, %xmm0
; X64-SSE2-NEXT: movd %xmm0, %eax
; X64-SSE2-NEXT: retq
;
@@ -2022,36 +2022,36 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X86-SSE4-NEXT: andl $-16, %esp
; X86-SSE4-NEXT: subl $16, %esp
; X86-SSE4-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE4-NEXT: pmulld 56(%ebp), %xmm2
-; X86-SSE4-NEXT: pmulld 24(%ebp), %xmm0
-; X86-SSE4-NEXT: pmulld %xmm2, %xmm0
; X86-SSE4-NEXT: pmulld 72(%ebp), %xmm3
; X86-SSE4-NEXT: pmulld 40(%ebp), %xmm1
; X86-SSE4-NEXT: pmulld %xmm3, %xmm1
-; X86-SSE4-NEXT: pmulld %xmm0, %xmm1
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE4-NEXT: pmulld 56(%ebp), %xmm2
+; X86-SSE4-NEXT: pmulld 24(%ebp), %xmm0
+; X86-SSE4-NEXT: pmulld %xmm2, %xmm0
; X86-SSE4-NEXT: pmulld %xmm1, %xmm0
-; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-SSE4-NEXT: pmulld %xmm0, %xmm1
-; X86-SSE4-NEXT: movd %xmm1, %eax
+; X86-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X86-SSE4-NEXT: pmulld %xmm1, %xmm0
+; X86-SSE4-NEXT: movd %xmm0, %eax
; X86-SSE4-NEXT: movl %ebp, %esp
; X86-SSE4-NEXT: popl %ebp
; X86-SSE4-NEXT: retl
;
; X64-SSE4-LABEL: test_v32i32:
; X64-SSE4: # %bb.0:
-; X64-SSE4-NEXT: pmulld %xmm6, %xmm2
-; X64-SSE4-NEXT: pmulld %xmm4, %xmm0
-; X64-SSE4-NEXT: pmulld %xmm2, %xmm0
; X64-SSE4-NEXT: pmulld %xmm7, %xmm3
; X64-SSE4-NEXT: pmulld %xmm5, %xmm1
; X64-SSE4-NEXT: pmulld %xmm3, %xmm1
-; X64-SSE4-NEXT: pmulld %xmm0, %xmm1
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE4-NEXT: pmulld %xmm6, %xmm2
+; X64-SSE4-NEXT: pmulld %xmm4, %xmm0
+; X64-SSE4-NEXT: pmulld %xmm2, %xmm0
; X64-SSE4-NEXT: pmulld %xmm1, %xmm0
-; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-SSE4-NEXT: pmulld %xmm0, %xmm1
-; X64-SSE4-NEXT: movd %xmm1, %eax
+; X64-SSE4-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; X64-SSE4-NEXT: pmulld %xmm1, %xmm0
+; X64-SSE4-NEXT: movd %xmm0, %eax
; X64-SSE4-NEXT: retq
;
; X86-AVX1-LABEL: test_v32i32:
@@ -2060,16 +2060,16 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X86-AVX1-NEXT: movl %esp, %ebp
; X86-AVX1-NEXT: andl $-32, %esp
; X86-AVX1-NEXT: subl $32, %esp
-; X86-AVX1-NEXT: vpmulld %xmm2, %xmm0, %xmm3
-; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-NEXT: vpmulld %xmm3, %xmm4, %xmm3
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-NEXT: vpmulld 24(%ebp), %xmm4, %xmm4
+; X86-AVX1-NEXT: vpmulld %xmm4, %xmm3, %xmm3
; X86-AVX1-NEXT: vpmulld %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpmulld 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-NEXT: vpmulld %xmm2, %xmm3, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-NEXT: vpmulld 24(%ebp), %xmm1, %xmm1
+; X86-AVX1-NEXT: vpmulld 8(%ebp), %xmm1, %xmm1
; X86-AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpmulld %xmm0, %xmm2, %xmm0
+; X86-AVX1-NEXT: vpmulld %xmm3, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -2082,17 +2082,17 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
;
; X64-AVX1-LABEL: test_v32i32:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vpmulld %xmm3, %xmm1, %xmm4
-; X64-AVX1-NEXT: vpmulld %xmm2, %xmm0, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-NEXT: vpmulld %xmm4, %xmm5, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-NEXT: vpmulld %xmm5, %xmm6, %xmm5
; X64-AVX1-NEXT: vpmulld %xmm4, %xmm5, %xmm4
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-NEXT: vpmulld %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-NEXT: vpmulld %xmm2, %xmm0, %xmm0
; X64-AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpmulld %xmm0, %xmm4, %xmm0
+; X64-AVX1-NEXT: vpmulld %xmm4, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpmulld %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -2141,7 +2141,7 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; AVX512: # %bb.0:
; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpmulld %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512-NEXT: vpmulld %xmm1, %xmm0, %xmm0
; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -2370,90 +2370,22 @@ define i16 @test_v32i16(<32 x i16> %a0) nounwind {
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: ret{{[l|q]}}
;
-; AVX512F-LABEL: test_v32i16:
-; AVX512F: # %bb.0:
-; AVX512F-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512F-NEXT: vpmullw %ymm1, %ymm0, %ymm0
-; AVX512F-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512F-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512F-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512F-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512F-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vmovd %xmm0, %eax
-; AVX512F-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512F-NEXT: vzeroupper
-; AVX512F-NEXT: retq
-;
-; AVX512BW-LABEL: test_v32i16:
-; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vmovd %xmm0, %eax
-; AVX512BW-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512BW-NEXT: vzeroupper
-; AVX512BW-NEXT: retq
-;
-; AVX512BWVL-LABEL: test_v32i16:
-; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
-; AVX512BWVL-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vmovd %xmm0, %eax
-; AVX512BWVL-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512BWVL-NEXT: vzeroupper
-; AVX512BWVL-NEXT: retq
-;
-; AVX512DQ-LABEL: test_v32i16:
-; AVX512DQ: # %bb.0:
-; AVX512DQ-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512DQ-NEXT: vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQ-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512DQ-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQ-NEXT: vmovd %xmm0, %eax
-; AVX512DQ-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512DQ-NEXT: vzeroupper
-; AVX512DQ-NEXT: retq
-;
-; AVX512DQVL-LABEL: test_v32i16:
-; AVX512DQVL: # %bb.0:
-; AVX512DQVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512DQVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
-; AVX512DQVL-NEXT: vextracti128 $1, %ymm0, %xmm1
-; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT: vpsrld $16, %xmm0, %xmm1
-; AVX512DQVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; AVX512DQVL-NEXT: vmovd %xmm0, %eax
-; AVX512DQVL-NEXT: # kill: def $ax killed $ax killed $eax
-; AVX512DQVL-NEXT: vzeroupper
-; AVX512DQVL-NEXT: retq
+; AVX512-LABEL: test_v32i16:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512-NEXT: vpmullw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX512-NEXT: vpmullw %xmm1, %xmm0, %xmm0
+; AVX512-NEXT: vmovd %xmm0, %eax
+; AVX512-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512-NEXT: vzeroupper
+; AVX512-NEXT: retq
%1 = call i16 @llvm.vector.reduce.mul.v32i16(<32 x i16> %a0)
ret i16 %1
}
@@ -2466,21 +2398,21 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
; X86-SSE-NEXT: andl $-16, %esp
; X86-SSE-NEXT: subl $16, %esp
; X86-SSE-NEXT: movdqa 8(%ebp), %xmm3
-; X86-SSE-NEXT: pmullw 56(%ebp), %xmm2
-; X86-SSE-NEXT: pmullw 24(%ebp), %xmm0
-; X86-SSE-NEXT: pmullw %xmm2, %xmm0
; X86-SSE-NEXT: pmullw 72(%ebp), %xmm3
; X86-SSE-NEXT: pmullw 40(%ebp), %xmm1
; X86-SSE-NEXT: pmullw %xmm3, %xmm1
-; X86-SSE-NEXT: pmullw %xmm0, %xmm1
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X86-SSE-NEXT: pmullw 56(%ebp), %xmm2
+; X86-SSE-NEXT: pmullw 24(%ebp), %xmm0
+; X86-SSE-NEXT: pmullw %xmm2, %xmm0
; X86-SSE-NEXT: pmullw %xmm1, %xmm0
-; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-SSE-NEXT: pmullw %xmm0, %xmm1
-; X86-SSE-NEXT: movdqa %xmm1, %xmm0
-; X86-SSE-NEXT: psrld $16, %xmm0
+; X86-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; X86-SSE-NEXT: pmullw %xmm1, %xmm0
-; X86-SSE-NEXT: movd %xmm0, %eax
+; X86-SSE-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE-NEXT: psrld $16, %xmm1
+; X86-SSE-NEXT: pmullw %xmm0, %xmm1
+; X86-SSE-NEXT: movd %xmm1, %eax
; X86-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X86-SSE-NEXT: movl %ebp, %esp
; X86-SSE-NEXT: popl %ebp
@@ -2488,21 +2420,21 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
;
; X64-SSE-LABEL: test_v64i16:
; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pmullw %xmm6, %xmm2
-; X64-SSE-NEXT: pmullw %xmm4, %xmm0
-; X64-SSE-NEXT: pmullw %xmm2, %xmm0
; X64-SSE-NEXT: pmullw %xmm7, %xmm3
; X64-SSE-NEXT: pmullw %xmm5, %xmm1
; X64-SSE-NEXT: pmullw %xmm3, %xmm1
-; X64-SSE-NEXT: pmullw %xmm0, %xmm1
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; X64-SSE-NEXT: pmullw %xmm6, %xmm2
+; X64-SSE-NEXT: pmullw %xmm4, %xmm0
+; X64-SSE-NEXT: pmullw %xmm2, %xmm0
; X64-SSE-NEXT: pmullw %xmm1, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-SSE-NEXT: pmullw %xmm0, %xmm1
-; X64-SSE-NEXT: movdqa %xmm1, %xmm0
-; X64-SSE-NEXT: psrld $16, %xmm0
+; X64-SSE-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
; X64-SSE-NEXT: pmullw %xmm1, %xmm0
-; X64-SSE-NEXT: movd %xmm0, %eax
+; X64-SSE-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE-NEXT: psrld $16, %xmm1
+; X64-SSE-NEXT: pmullw %xmm0, %xmm1
+; X64-SSE-NEXT: movd %xmm1, %eax
; X64-SSE-NEXT: # kill: def $ax killed $ax killed $eax
; X64-SSE-NEXT: retq
;
@@ -2512,16 +2444,16 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
; X86-AVX1-NEXT: movl %esp, %ebp
; X86-AVX1-NEXT: andl $-32, %esp
; X86-AVX1-NEXT: subl $32, %esp
-; X86-AVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm3
-; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
+; X86-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm3
+; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm4
+; X86-AVX1-NEXT: vpmullw %xmm3, %xmm4, %xmm3
+; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm4
+; X86-AVX1-NEXT: vpmullw 24(%ebp), %xmm4, %xmm4
+; X86-AVX1-NEXT: vpmullw %xmm4, %xmm3, %xmm3
; X86-AVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpmullw 8(%ebp), %xmm1, %xmm2
-; X86-AVX1-NEXT: vpmullw %xmm2, %xmm3, %xmm2
-; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
-; X86-AVX1-NEXT: vpmullw 24(%ebp), %xmm1, %xmm1
+; X86-AVX1-NEXT: vpmullw 8(%ebp), %xmm1, %xmm1
; X86-AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; X86-AVX1-NEXT: vpmullw %xmm0, %xmm2, %xmm0
+; X86-AVX1-NEXT: vpmullw %xmm3, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X86-AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -2537,17 +2469,17 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
;
; X64-AVX1-LABEL: test_v64i16:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vpmullw %xmm3, %xmm1, %xmm4
-; X64-AVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm5
+; X64-AVX1-NEXT: vpmullw %xmm4, %xmm5, %xmm4
+; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm5
+; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm6
+; X64-AVX1-NEXT: vpmullw %xmm5, %xmm6, %xmm5
; X64-AVX1-NEXT: vpmullw %xmm4, %xmm5, %xmm4
-; X64-AVX1-NEXT: vextractf128 $1, %ymm3, %xmm3
-; X64-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1
; X64-AVX1-NEXT: vpmullw %xmm3, %xmm1, %xmm1
-; X64-AVX1-NEXT: vextractf128 $1, %ymm2, %xmm2
-; X64-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm0
; X64-AVX1-NEXT: vpmullw %xmm2, %xmm0, %xmm0
; X64-AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpmullw %xmm0, %xmm4, %xmm0
+; X64-AVX1-NEXT: vpmullw %xmm4, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; X64-AVX1-NEXT: vpmullw %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
@@ -2625,7 +2557,7 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
; AVX512BW: # %bb.0:
; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BW-NEXT: vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vpmullw %ymm1, %ymm0, %ymm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512BW-NEXT: vpmullw %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -2643,7 +2575,7 @@ define i16 @test_v64i16(<64 x i16> %a0) nounwind {
; AVX512BWVL: # %bb.0:
; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
; AVX512BWVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512BWVL-NEXT: vpmullw %zmm1, %zmm0, %zmm0
+; AVX512BWVL-NEXT: vpmullw %ymm1, %ymm0, %ymm0
; AVX512BWVL-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512BWVL-NEXT: vpmullw %xmm1, %xmm0, %xmm0
; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
index acddf254c2998..df8c2b7e9575b 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll
@@ -49,14 +49,8 @@ for.end:
define signext i16 @mul_v8i16(ptr %p) {
; CHECK-LABEL: @mul_v8i16(
; CHECK-NEXT: entry:
-; CHECK-NEXT: [[TMP1:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 2, !tbaa [[TBAA4:![0-9]+]]
-; CHECK-NEXT: [[RDX_SHUF:%.*]] = shufflevector <8 x i16> [[TMP1]], <8 x i16> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[BIN_RDX:%.*]] = mul <8 x i16> [[TMP1]], [[RDX_SHUF]]
-; CHECK-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <8 x i16> [[BIN_RDX]], <8 x i16> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[BIN_RDX4:%.*]] = mul <8 x i16> [[BIN_RDX]], [[RDX_SHUF3]]
-; CHECK-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <8 x i16> [[BIN_RDX4]], <8 x i16> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
-; CHECK-NEXT: [[BIN_RDX6:%.*]] = mul <8 x i16> [[BIN_RDX4]], [[RDX_SHUF5]]
-; CHECK-NEXT: [[TMP2:%.*]] = extractelement <8 x i16> [[BIN_RDX6]], i32 0
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x i16>, ptr [[P:%.*]], align 2, !tbaa [[TBAA4:![0-9]+]]
+; CHECK-NEXT: [[TMP2:%.*]] = tail call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> [[TMP0]])
; CHECK-NEXT: ret i16 [[TMP2]]
;
entry:
>From a49430494ae24047c519fcd45f89533a2b2656df Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev at redking.me.uk>
Date: Mon, 6 Jul 2026 10:03:02 +0100
Subject: [PATCH 2/2] Use default scalarization
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 4 +-
.../X86/avx512-intrinsics-fast-isel.ll | 22 +-
llvm/test/CodeGen/X86/vector-reduce-mul.ll | 840 +++++++++---------
3 files changed, 404 insertions(+), 462 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 8888070f9234a..d3c6363930806 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -34354,8 +34354,8 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::VECREDUCE_UMIN: return LowerMINMAX_REDUCE(Op, Subtarget, DAG);
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
- case ISD::VECREDUCE_XOR: return LowerVECREDUCE(Op, Subtarget, DAG, true);
- case ISD::VECREDUCE_MUL: return LowerVECREDUCE(Op, Subtarget, DAG, false);
+ case ISD::VECREDUCE_XOR:
+ case ISD::VECREDUCE_MUL: return LowerVECREDUCE(Op, Subtarget, DAG, true);
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
case ISD::FMINIMUMNUM:
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
index ddd94bfd3e4b6..ecbc38d968f8d 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
@@ -6600,16 +6600,9 @@ define i64 @test_mm512_reduce_mul_epi64(<8 x i64> %__W) nounwind {
; X64-NEXT: vpsllq $32, %xmm2, %xmm2
; X64-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; X64-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X64-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-NEXT: vpsrlq $32, %xmm0, %xmm2
-; X64-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; X64-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; X64-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X64-NEXT: vpsllq $32, %xmm2, %xmm2
-; X64-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; X64-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-NEXT: vpextrq $1, %xmm0, %rcx
; X64-NEXT: vmovq %xmm0, %rax
+; X64-NEXT: imulq %rcx, %rax
; X64-NEXT: vzeroupper
; X64-NEXT: retq
entry:
@@ -6776,16 +6769,9 @@ define i64 @test_mm512_mask_reduce_mul_epi64(i8 zeroext %__M, <8 x i64> %__W) no
; X64-NEXT: vpsllq $32, %xmm2, %xmm2
; X64-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; X64-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X64-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-NEXT: vpsrlq $32, %xmm0, %xmm2
-; X64-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; X64-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; X64-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X64-NEXT: vpsllq $32, %xmm2, %xmm2
-; X64-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; X64-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-NEXT: vpextrq $1, %xmm0, %rcx
; X64-NEXT: vmovq %xmm0, %rax
+; X64-NEXT: imulq %rcx, %rax
; X64-NEXT: vzeroupper
; X64-NEXT: retq
entry:
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index b1d5b188b8de2..edf5022a64944 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -38,21 +38,13 @@ define i64 @test_v2i64(<2 x i64> %a0) nounwind {
; X86-SSE2-NEXT: movd %xmm0, %edx
; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v2i64:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE-NEXT: psrlq $32, %xmm2
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm2
-; X64-SSE-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-SSE-NEXT: pmuludq %xmm0, %xmm3
-; X64-SSE-NEXT: paddq %xmm2, %xmm3
-; X64-SSE-NEXT: psllq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm0
-; X64-SSE-NEXT: paddq %xmm3, %xmm0
-; X64-SSE-NEXT: movq %xmm0, %rax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v2i64:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: movq %xmm0, %rcx
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: imulq %rcx, %rax
+; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v2i64:
; X86-SSE4: # %bb.0:
@@ -71,6 +63,13 @@ define i64 @test_v2i64(<2 x i64> %a0) nounwind {
; X86-SSE4-NEXT: pextrd $1, %xmm0, %edx
; X86-SSE4-NEXT: retl
;
+; X64-SSE4-LABEL: test_v2i64:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: pextrq $1, %xmm0, %rcx
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: imulq %rcx, %rax
+; X64-SSE4-NEXT: retq
+;
; X86-AVX1-LABEL: test_v2i64:
; X86-AVX1: # %bb.0:
; X86-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -88,16 +87,9 @@ define i64 @test_v2i64(<2 x i64> %a0) nounwind {
;
; X64-AVX1-LABEL: test_v2i64:
; X64-AVX1: # %bb.0:
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2
-; X64-AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X64-AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpextrq $1, %xmm0, %rcx
; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: imulq %rcx, %rax
; X64-AVX1-NEXT: retq
;
; X86-AVX2-LABEL: test_v2i64:
@@ -117,58 +109,30 @@ define i64 @test_v2i64(<2 x i64> %a0) nounwind {
;
; X64-AVX2-LABEL: test_v2i64:
; X64-AVX2: # %bb.0:
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2
-; X64-AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; X64-AVX2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; X64-AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X64-AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
-; X64-AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpextrq $1, %xmm0, %rcx
; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: imulq %rcx, %rax
; X64-AVX2-NEXT: retq
;
; AVX512F-LABEL: test_v2i64:
; AVX512F: # %bb.0:
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512F-NEXT: vpsrlq $32, %xmm0, %xmm2
-; AVX512F-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; AVX512F-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX512F-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT: vpsllq $32, %xmm2, %xmm2
-; AVX512F-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512F-NEXT: vmovq %xmm0, %rax
+; AVX512F-NEXT: imulq %rcx, %rax
; AVX512F-NEXT: retq
;
; AVX512BW-LABEL: test_v2i64:
; AVX512BW: # %bb.0:
-; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2
-; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; AVX512BW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
-; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512BW-NEXT: vmovq %xmm0, %rax
+; AVX512BW-NEXT: imulq %rcx, %rax
; AVX512BW-NEXT: retq
;
; AVX512BWVL-LABEL: test_v2i64:
; AVX512BWVL: # %bb.0:
-; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2
-; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; AVX512BWVL-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
-; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; AVX512BWVL-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512BWVL-NEXT: vmovq %xmm0, %rax
+; AVX512BWVL-NEXT: imulq %rcx, %rax
; AVX512BWVL-NEXT: retq
;
; AVX512DQ-LABEL: test_v2i64:
@@ -219,31 +183,23 @@ define i64 @test_v4i64(<4 x i64> %a0) nounwind {
; X86-SSE2-NEXT: movd %xmm0, %edx
; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v4i64:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE-NEXT: psrlq $32, %xmm2
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm2
-; X64-SSE-NEXT: movdqa %xmm1, %xmm3
-; X64-SSE-NEXT: psrlq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm0, %xmm3
-; X64-SSE-NEXT: paddq %xmm2, %xmm3
-; X64-SSE-NEXT: psllq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm0
-; X64-SSE-NEXT: paddq %xmm3, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE-NEXT: psrlq $32, %xmm2
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm2
-; X64-SSE-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-SSE-NEXT: pmuludq %xmm0, %xmm3
-; X64-SSE-NEXT: paddq %xmm2, %xmm3
-; X64-SSE-NEXT: psllq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm0
-; X64-SSE-NEXT: paddq %xmm3, %xmm0
-; X64-SSE-NEXT: movq %xmm0, %rax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v4i64:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm2
+; X64-SSE2-NEXT: psrlq $32, %xmm2
+; X64-SSE2-NEXT: pmuludq %xmm1, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE2-NEXT: psrlq $32, %xmm3
+; X64-SSE2-NEXT: pmuludq %xmm0, %xmm3
+; X64-SSE2-NEXT: paddq %xmm2, %xmm3
+; X64-SSE2-NEXT: psllq $32, %xmm3
+; X64-SSE2-NEXT: pmuludq %xmm1, %xmm0
+; X64-SSE2-NEXT: paddq %xmm3, %xmm0
+; X64-SSE2-NEXT: movq %xmm0, %rcx
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: imulq %rcx, %rax
+; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v4i64:
; X86-SSE4: # %bb.0:
@@ -272,6 +228,23 @@ define i64 @test_v4i64(<4 x i64> %a0) nounwind {
; X86-SSE4-NEXT: pextrd $1, %xmm0, %edx
; X86-SSE4-NEXT: retl
;
+; X64-SSE4-LABEL: test_v4i64:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X64-SSE4-NEXT: psrlq $32, %xmm2
+; X64-SSE4-NEXT: pmuludq %xmm1, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE4-NEXT: psrlq $32, %xmm3
+; X64-SSE4-NEXT: pmuludq %xmm0, %xmm3
+; X64-SSE4-NEXT: paddq %xmm2, %xmm3
+; X64-SSE4-NEXT: psllq $32, %xmm3
+; X64-SSE4-NEXT: pmuludq %xmm1, %xmm0
+; X64-SSE4-NEXT: paddq %xmm3, %xmm0
+; X64-SSE4-NEXT: pextrq $1, %xmm0, %rcx
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: imulq %rcx, %rax
+; X64-SSE4-NEXT: retq
+;
; X86-AVX1-LABEL: test_v4i64:
; X86-AVX1: # %bb.0:
; X86-AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1
@@ -308,16 +281,9 @@ define i64 @test_v4i64(<4 x i64> %a0) nounwind {
; X64-AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
; X64-AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2
-; X64-AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X64-AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpextrq $1, %xmm0, %rcx
; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: imulq %rcx, %rax
; X64-AVX1-NEXT: vzeroupper
; X64-AVX1-NEXT: retq
;
@@ -357,16 +323,9 @@ define i64 @test_v4i64(<4 x i64> %a0) nounwind {
; X64-AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
; X64-AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2
-; X64-AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; X64-AVX2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; X64-AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X64-AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
-; X64-AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpextrq $1, %xmm0, %rcx
; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: imulq %rcx, %rax
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
@@ -381,16 +340,9 @@ define i64 @test_v4i64(<4 x i64> %a0) nounwind {
; AVX512F-NEXT: vpsllq $32, %xmm2, %xmm2
; AVX512F-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512F-NEXT: vpsrlq $32, %xmm0, %xmm2
-; AVX512F-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; AVX512F-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX512F-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT: vpsllq $32, %xmm2, %xmm2
-; AVX512F-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512F-NEXT: vmovq %xmm0, %rax
+; AVX512F-NEXT: imulq %rcx, %rax
; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
@@ -405,16 +357,9 @@ define i64 @test_v4i64(<4 x i64> %a0) nounwind {
; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2
-; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; AVX512BW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
-; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512BW-NEXT: vmovq %xmm0, %rax
+; AVX512BW-NEXT: imulq %rcx, %rax
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -429,16 +374,9 @@ define i64 @test_v4i64(<4 x i64> %a0) nounwind {
; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2
-; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; AVX512BWVL-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
-; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; AVX512BWVL-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512BWVL-NEXT: vmovq %xmm0, %rax
+; AVX512BWVL-NEXT: imulq %rcx, %rax
; AVX512BWVL-NEXT: vzeroupper
; AVX512BWVL-NEXT: retq
;
@@ -522,51 +460,43 @@ define i64 @test_v8i64(<8 x i64> %a0) nounwind {
; X86-SSE2-NEXT: popl %ebp
; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v8i64:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: movdqa %xmm1, %xmm4
-; X64-SSE-NEXT: psrlq $32, %xmm4
-; X64-SSE-NEXT: pmuludq %xmm3, %xmm4
-; X64-SSE-NEXT: movdqa %xmm3, %xmm5
-; X64-SSE-NEXT: psrlq $32, %xmm5
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm5
-; X64-SSE-NEXT: paddq %xmm4, %xmm5
-; X64-SSE-NEXT: psllq $32, %xmm5
-; X64-SSE-NEXT: pmuludq %xmm3, %xmm1
-; X64-SSE-NEXT: paddq %xmm5, %xmm1
-; X64-SSE-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE-NEXT: psrlq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm2, %xmm3
-; X64-SSE-NEXT: movdqa %xmm2, %xmm4
-; X64-SSE-NEXT: psrlq $32, %xmm4
-; X64-SSE-NEXT: pmuludq %xmm0, %xmm4
-; X64-SSE-NEXT: paddq %xmm3, %xmm4
-; X64-SSE-NEXT: psllq $32, %xmm4
-; X64-SSE-NEXT: pmuludq %xmm2, %xmm0
-; X64-SSE-NEXT: paddq %xmm4, %xmm0
-; X64-SSE-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE-NEXT: psrlq $32, %xmm2
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm2
-; X64-SSE-NEXT: movdqa %xmm1, %xmm3
-; X64-SSE-NEXT: psrlq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm0, %xmm3
-; X64-SSE-NEXT: paddq %xmm2, %xmm3
-; X64-SSE-NEXT: psllq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm0
-; X64-SSE-NEXT: paddq %xmm3, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE-NEXT: psrlq $32, %xmm2
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm2
-; X64-SSE-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-SSE-NEXT: pmuludq %xmm0, %xmm3
-; X64-SSE-NEXT: paddq %xmm2, %xmm3
-; X64-SSE-NEXT: psllq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm0
-; X64-SSE-NEXT: paddq %xmm3, %xmm0
-; X64-SSE-NEXT: movq %xmm0, %rax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v8i64:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm4
+; X64-SSE2-NEXT: psrlq $32, %xmm4
+; X64-SSE2-NEXT: pmuludq %xmm3, %xmm4
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm5
+; X64-SSE2-NEXT: psrlq $32, %xmm5
+; X64-SSE2-NEXT: pmuludq %xmm1, %xmm5
+; X64-SSE2-NEXT: paddq %xmm4, %xmm5
+; X64-SSE2-NEXT: psllq $32, %xmm5
+; X64-SSE2-NEXT: pmuludq %xmm3, %xmm1
+; X64-SSE2-NEXT: paddq %xmm5, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE2-NEXT: psrlq $32, %xmm3
+; X64-SSE2-NEXT: pmuludq %xmm2, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm4
+; X64-SSE2-NEXT: psrlq $32, %xmm4
+; X64-SSE2-NEXT: pmuludq %xmm0, %xmm4
+; X64-SSE2-NEXT: paddq %xmm3, %xmm4
+; X64-SSE2-NEXT: psllq $32, %xmm4
+; X64-SSE2-NEXT: pmuludq %xmm2, %xmm0
+; X64-SSE2-NEXT: paddq %xmm4, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm2
+; X64-SSE2-NEXT: psrlq $32, %xmm2
+; X64-SSE2-NEXT: pmuludq %xmm1, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE2-NEXT: psrlq $32, %xmm3
+; X64-SSE2-NEXT: pmuludq %xmm0, %xmm3
+; X64-SSE2-NEXT: paddq %xmm2, %xmm3
+; X64-SSE2-NEXT: psllq $32, %xmm3
+; X64-SSE2-NEXT: pmuludq %xmm1, %xmm0
+; X64-SSE2-NEXT: paddq %xmm3, %xmm0
+; X64-SSE2-NEXT: movq %xmm0, %rcx
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: imulq %rcx, %rax
+; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v8i64:
; X86-SSE4: # %bb.0:
@@ -622,6 +552,43 @@ define i64 @test_v8i64(<8 x i64> %a0) nounwind {
; X86-SSE4-NEXT: popl %ebp
; X86-SSE4-NEXT: retl
;
+; X64-SSE4-LABEL: test_v8i64:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm4
+; X64-SSE4-NEXT: psrlq $32, %xmm4
+; X64-SSE4-NEXT: pmuludq %xmm3, %xmm4
+; X64-SSE4-NEXT: movdqa %xmm3, %xmm5
+; X64-SSE4-NEXT: psrlq $32, %xmm5
+; X64-SSE4-NEXT: pmuludq %xmm1, %xmm5
+; X64-SSE4-NEXT: paddq %xmm4, %xmm5
+; X64-SSE4-NEXT: psllq $32, %xmm5
+; X64-SSE4-NEXT: pmuludq %xmm3, %xmm1
+; X64-SSE4-NEXT: paddq %xmm5, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE4-NEXT: psrlq $32, %xmm3
+; X64-SSE4-NEXT: pmuludq %xmm2, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm2, %xmm4
+; X64-SSE4-NEXT: psrlq $32, %xmm4
+; X64-SSE4-NEXT: pmuludq %xmm0, %xmm4
+; X64-SSE4-NEXT: paddq %xmm3, %xmm4
+; X64-SSE4-NEXT: psllq $32, %xmm4
+; X64-SSE4-NEXT: pmuludq %xmm2, %xmm0
+; X64-SSE4-NEXT: paddq %xmm4, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X64-SSE4-NEXT: psrlq $32, %xmm2
+; X64-SSE4-NEXT: pmuludq %xmm1, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE4-NEXT: psrlq $32, %xmm3
+; X64-SSE4-NEXT: pmuludq %xmm0, %xmm3
+; X64-SSE4-NEXT: paddq %xmm2, %xmm3
+; X64-SSE4-NEXT: psllq $32, %xmm3
+; X64-SSE4-NEXT: pmuludq %xmm1, %xmm0
+; X64-SSE4-NEXT: paddq %xmm3, %xmm0
+; X64-SSE4-NEXT: pextrq $1, %xmm0, %rcx
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: imulq %rcx, %rax
+; X64-SSE4-NEXT: retq
+;
; X86-AVX1-LABEL: test_v8i64:
; X86-AVX1: # %bb.0:
; X86-AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2
@@ -692,16 +659,9 @@ define i64 @test_v8i64(<8 x i64> %a0) nounwind {
; X64-AVX1-NEXT: vpsllq $32, %xmm1, %xmm1
; X64-AVX1-NEXT: vpmuludq %xmm2, %xmm0, %xmm0
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2
-; X64-AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X64-AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpextrq $1, %xmm0, %rcx
; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: imulq %rcx, %rax
; X64-AVX1-NEXT: vzeroupper
; X64-AVX1-NEXT: retq
;
@@ -757,16 +717,9 @@ define i64 @test_v8i64(<8 x i64> %a0) nounwind {
; X64-AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
; X64-AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2
-; X64-AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; X64-AVX2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; X64-AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X64-AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
-; X64-AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpextrq $1, %xmm0, %rcx
; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: imulq %rcx, %rax
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
@@ -790,16 +743,9 @@ define i64 @test_v8i64(<8 x i64> %a0) nounwind {
; AVX512F-NEXT: vpsllq $32, %xmm2, %xmm2
; AVX512F-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512F-NEXT: vpsrlq $32, %xmm0, %xmm2
-; AVX512F-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; AVX512F-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX512F-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT: vpsllq $32, %xmm2, %xmm2
-; AVX512F-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512F-NEXT: vmovq %xmm0, %rax
+; AVX512F-NEXT: imulq %rcx, %rax
; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
@@ -823,16 +769,9 @@ define i64 @test_v8i64(<8 x i64> %a0) nounwind {
; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2
-; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; AVX512BW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
-; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512BW-NEXT: vmovq %xmm0, %rax
+; AVX512BW-NEXT: imulq %rcx, %rax
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -856,16 +795,9 @@ define i64 @test_v8i64(<8 x i64> %a0) nounwind {
; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2
-; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; AVX512BWVL-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
-; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; AVX512BWVL-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512BWVL-NEXT: vmovq %xmm0, %rax
+; AVX512BWVL-NEXT: imulq %rcx, %rax
; AVX512BWVL-NEXT: vzeroupper
; AVX512BWVL-NEXT: retq
;
@@ -996,91 +928,83 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: popl %ebp
; X86-SSE2-NEXT: retl
;
-; X64-SSE-LABEL: test_v16i64:
-; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: movdqa %xmm3, %xmm8
-; X64-SSE-NEXT: psrlq $32, %xmm8
-; X64-SSE-NEXT: pmuludq %xmm7, %xmm8
-; X64-SSE-NEXT: movdqa %xmm7, %xmm9
-; X64-SSE-NEXT: psrlq $32, %xmm9
-; X64-SSE-NEXT: pmuludq %xmm3, %xmm9
-; X64-SSE-NEXT: paddq %xmm8, %xmm9
-; X64-SSE-NEXT: psllq $32, %xmm9
-; X64-SSE-NEXT: pmuludq %xmm7, %xmm3
-; X64-SSE-NEXT: paddq %xmm9, %xmm3
-; X64-SSE-NEXT: movdqa %xmm1, %xmm7
-; X64-SSE-NEXT: psrlq $32, %xmm7
-; X64-SSE-NEXT: pmuludq %xmm5, %xmm7
-; X64-SSE-NEXT: movdqa %xmm5, %xmm8
-; X64-SSE-NEXT: psrlq $32, %xmm8
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm8
-; X64-SSE-NEXT: paddq %xmm7, %xmm8
-; X64-SSE-NEXT: psllq $32, %xmm8
-; X64-SSE-NEXT: pmuludq %xmm5, %xmm1
-; X64-SSE-NEXT: paddq %xmm8, %xmm1
-; X64-SSE-NEXT: movdqa %xmm1, %xmm5
-; X64-SSE-NEXT: psrlq $32, %xmm5
-; X64-SSE-NEXT: pmuludq %xmm3, %xmm5
-; X64-SSE-NEXT: movdqa %xmm3, %xmm7
-; X64-SSE-NEXT: psrlq $32, %xmm7
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm7
-; X64-SSE-NEXT: paddq %xmm5, %xmm7
-; X64-SSE-NEXT: psllq $32, %xmm7
-; X64-SSE-NEXT: pmuludq %xmm3, %xmm1
-; X64-SSE-NEXT: paddq %xmm7, %xmm1
-; X64-SSE-NEXT: movdqa %xmm2, %xmm3
-; X64-SSE-NEXT: psrlq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm6, %xmm3
-; X64-SSE-NEXT: movdqa %xmm6, %xmm5
-; X64-SSE-NEXT: psrlq $32, %xmm5
-; X64-SSE-NEXT: pmuludq %xmm2, %xmm5
-; X64-SSE-NEXT: paddq %xmm3, %xmm5
-; X64-SSE-NEXT: psllq $32, %xmm5
-; X64-SSE-NEXT: pmuludq %xmm6, %xmm2
-; X64-SSE-NEXT: paddq %xmm5, %xmm2
-; X64-SSE-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE-NEXT: psrlq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm4, %xmm3
-; X64-SSE-NEXT: movdqa %xmm4, %xmm5
-; X64-SSE-NEXT: psrlq $32, %xmm5
-; X64-SSE-NEXT: pmuludq %xmm0, %xmm5
-; X64-SSE-NEXT: paddq %xmm3, %xmm5
-; X64-SSE-NEXT: psllq $32, %xmm5
-; X64-SSE-NEXT: pmuludq %xmm4, %xmm0
-; X64-SSE-NEXT: paddq %xmm5, %xmm0
-; X64-SSE-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE-NEXT: psrlq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm2, %xmm3
-; X64-SSE-NEXT: movdqa %xmm2, %xmm4
-; X64-SSE-NEXT: psrlq $32, %xmm4
-; X64-SSE-NEXT: pmuludq %xmm0, %xmm4
-; X64-SSE-NEXT: paddq %xmm3, %xmm4
-; X64-SSE-NEXT: psllq $32, %xmm4
-; X64-SSE-NEXT: pmuludq %xmm2, %xmm0
-; X64-SSE-NEXT: paddq %xmm4, %xmm0
-; X64-SSE-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE-NEXT: psrlq $32, %xmm2
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm2
-; X64-SSE-NEXT: movdqa %xmm1, %xmm3
-; X64-SSE-NEXT: psrlq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm0, %xmm3
-; X64-SSE-NEXT: paddq %xmm2, %xmm3
-; X64-SSE-NEXT: psllq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm0
-; X64-SSE-NEXT: paddq %xmm3, %xmm0
-; X64-SSE-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE-NEXT: movdqa %xmm0, %xmm2
-; X64-SSE-NEXT: psrlq $32, %xmm2
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm2
-; X64-SSE-NEXT: movdqa %xmm0, %xmm3
-; X64-SSE-NEXT: psrldq {{.*#+}} xmm3 = xmm3[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-SSE-NEXT: pmuludq %xmm0, %xmm3
-; X64-SSE-NEXT: paddq %xmm2, %xmm3
-; X64-SSE-NEXT: psllq $32, %xmm3
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm0
-; X64-SSE-NEXT: paddq %xmm3, %xmm0
-; X64-SSE-NEXT: movq %xmm0, %rax
-; X64-SSE-NEXT: retq
+; X64-SSE2-LABEL: test_v16i64:
+; X64-SSE2: # %bb.0:
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm8
+; X64-SSE2-NEXT: psrlq $32, %xmm8
+; X64-SSE2-NEXT: pmuludq %xmm7, %xmm8
+; X64-SSE2-NEXT: movdqa %xmm7, %xmm9
+; X64-SSE2-NEXT: psrlq $32, %xmm9
+; X64-SSE2-NEXT: pmuludq %xmm3, %xmm9
+; X64-SSE2-NEXT: paddq %xmm8, %xmm9
+; X64-SSE2-NEXT: psllq $32, %xmm9
+; X64-SSE2-NEXT: pmuludq %xmm7, %xmm3
+; X64-SSE2-NEXT: paddq %xmm9, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm7
+; X64-SSE2-NEXT: psrlq $32, %xmm7
+; X64-SSE2-NEXT: pmuludq %xmm5, %xmm7
+; X64-SSE2-NEXT: movdqa %xmm5, %xmm8
+; X64-SSE2-NEXT: psrlq $32, %xmm8
+; X64-SSE2-NEXT: pmuludq %xmm1, %xmm8
+; X64-SSE2-NEXT: paddq %xmm7, %xmm8
+; X64-SSE2-NEXT: psllq $32, %xmm8
+; X64-SSE2-NEXT: pmuludq %xmm5, %xmm1
+; X64-SSE2-NEXT: paddq %xmm8, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm5
+; X64-SSE2-NEXT: psrlq $32, %xmm5
+; X64-SSE2-NEXT: pmuludq %xmm3, %xmm5
+; X64-SSE2-NEXT: movdqa %xmm3, %xmm7
+; X64-SSE2-NEXT: psrlq $32, %xmm7
+; X64-SSE2-NEXT: pmuludq %xmm1, %xmm7
+; X64-SSE2-NEXT: paddq %xmm5, %xmm7
+; X64-SSE2-NEXT: psllq $32, %xmm7
+; X64-SSE2-NEXT: pmuludq %xmm3, %xmm1
+; X64-SSE2-NEXT: paddq %xmm7, %xmm1
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm3
+; X64-SSE2-NEXT: psrlq $32, %xmm3
+; X64-SSE2-NEXT: pmuludq %xmm6, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm6, %xmm5
+; X64-SSE2-NEXT: psrlq $32, %xmm5
+; X64-SSE2-NEXT: pmuludq %xmm2, %xmm5
+; X64-SSE2-NEXT: paddq %xmm3, %xmm5
+; X64-SSE2-NEXT: psllq $32, %xmm5
+; X64-SSE2-NEXT: pmuludq %xmm6, %xmm2
+; X64-SSE2-NEXT: paddq %xmm5, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE2-NEXT: psrlq $32, %xmm3
+; X64-SSE2-NEXT: pmuludq %xmm4, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm4, %xmm5
+; X64-SSE2-NEXT: psrlq $32, %xmm5
+; X64-SSE2-NEXT: pmuludq %xmm0, %xmm5
+; X64-SSE2-NEXT: paddq %xmm3, %xmm5
+; X64-SSE2-NEXT: psllq $32, %xmm5
+; X64-SSE2-NEXT: pmuludq %xmm4, %xmm0
+; X64-SSE2-NEXT: paddq %xmm5, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE2-NEXT: psrlq $32, %xmm3
+; X64-SSE2-NEXT: pmuludq %xmm2, %xmm3
+; X64-SSE2-NEXT: movdqa %xmm2, %xmm4
+; X64-SSE2-NEXT: psrlq $32, %xmm4
+; X64-SSE2-NEXT: pmuludq %xmm0, %xmm4
+; X64-SSE2-NEXT: paddq %xmm3, %xmm4
+; X64-SSE2-NEXT: psllq $32, %xmm4
+; X64-SSE2-NEXT: pmuludq %xmm2, %xmm0
+; X64-SSE2-NEXT: paddq %xmm4, %xmm0
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm2
+; X64-SSE2-NEXT: psrlq $32, %xmm2
+; X64-SSE2-NEXT: pmuludq %xmm1, %xmm2
+; X64-SSE2-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE2-NEXT: psrlq $32, %xmm3
+; X64-SSE2-NEXT: pmuludq %xmm0, %xmm3
+; X64-SSE2-NEXT: paddq %xmm2, %xmm3
+; X64-SSE2-NEXT: psllq $32, %xmm3
+; X64-SSE2-NEXT: pmuludq %xmm1, %xmm0
+; X64-SSE2-NEXT: paddq %xmm3, %xmm0
+; X64-SSE2-NEXT: movq %xmm0, %rcx
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; X64-SSE2-NEXT: movq %xmm0, %rax
+; X64-SSE2-NEXT: imulq %rcx, %rax
+; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v16i64:
; X86-SSE4: # %bb.0:
@@ -1180,6 +1104,83 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE4-NEXT: popl %ebp
; X86-SSE4-NEXT: retl
;
+; X64-SSE4-LABEL: test_v16i64:
+; X64-SSE4: # %bb.0:
+; X64-SSE4-NEXT: movdqa %xmm3, %xmm8
+; X64-SSE4-NEXT: psrlq $32, %xmm8
+; X64-SSE4-NEXT: pmuludq %xmm7, %xmm8
+; X64-SSE4-NEXT: movdqa %xmm7, %xmm9
+; X64-SSE4-NEXT: psrlq $32, %xmm9
+; X64-SSE4-NEXT: pmuludq %xmm3, %xmm9
+; X64-SSE4-NEXT: paddq %xmm8, %xmm9
+; X64-SSE4-NEXT: psllq $32, %xmm9
+; X64-SSE4-NEXT: pmuludq %xmm7, %xmm3
+; X64-SSE4-NEXT: paddq %xmm9, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm7
+; X64-SSE4-NEXT: psrlq $32, %xmm7
+; X64-SSE4-NEXT: pmuludq %xmm5, %xmm7
+; X64-SSE4-NEXT: movdqa %xmm5, %xmm8
+; X64-SSE4-NEXT: psrlq $32, %xmm8
+; X64-SSE4-NEXT: pmuludq %xmm1, %xmm8
+; X64-SSE4-NEXT: paddq %xmm7, %xmm8
+; X64-SSE4-NEXT: psllq $32, %xmm8
+; X64-SSE4-NEXT: pmuludq %xmm5, %xmm1
+; X64-SSE4-NEXT: paddq %xmm8, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm5
+; X64-SSE4-NEXT: psrlq $32, %xmm5
+; X64-SSE4-NEXT: pmuludq %xmm3, %xmm5
+; X64-SSE4-NEXT: movdqa %xmm3, %xmm7
+; X64-SSE4-NEXT: psrlq $32, %xmm7
+; X64-SSE4-NEXT: pmuludq %xmm1, %xmm7
+; X64-SSE4-NEXT: paddq %xmm5, %xmm7
+; X64-SSE4-NEXT: psllq $32, %xmm7
+; X64-SSE4-NEXT: pmuludq %xmm3, %xmm1
+; X64-SSE4-NEXT: paddq %xmm7, %xmm1
+; X64-SSE4-NEXT: movdqa %xmm2, %xmm3
+; X64-SSE4-NEXT: psrlq $32, %xmm3
+; X64-SSE4-NEXT: pmuludq %xmm6, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm6, %xmm5
+; X64-SSE4-NEXT: psrlq $32, %xmm5
+; X64-SSE4-NEXT: pmuludq %xmm2, %xmm5
+; X64-SSE4-NEXT: paddq %xmm3, %xmm5
+; X64-SSE4-NEXT: psllq $32, %xmm5
+; X64-SSE4-NEXT: pmuludq %xmm6, %xmm2
+; X64-SSE4-NEXT: paddq %xmm5, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE4-NEXT: psrlq $32, %xmm3
+; X64-SSE4-NEXT: pmuludq %xmm4, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm4, %xmm5
+; X64-SSE4-NEXT: psrlq $32, %xmm5
+; X64-SSE4-NEXT: pmuludq %xmm0, %xmm5
+; X64-SSE4-NEXT: paddq %xmm3, %xmm5
+; X64-SSE4-NEXT: psllq $32, %xmm5
+; X64-SSE4-NEXT: pmuludq %xmm4, %xmm0
+; X64-SSE4-NEXT: paddq %xmm5, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE4-NEXT: psrlq $32, %xmm3
+; X64-SSE4-NEXT: pmuludq %xmm2, %xmm3
+; X64-SSE4-NEXT: movdqa %xmm2, %xmm4
+; X64-SSE4-NEXT: psrlq $32, %xmm4
+; X64-SSE4-NEXT: pmuludq %xmm0, %xmm4
+; X64-SSE4-NEXT: paddq %xmm3, %xmm4
+; X64-SSE4-NEXT: psllq $32, %xmm4
+; X64-SSE4-NEXT: pmuludq %xmm2, %xmm0
+; X64-SSE4-NEXT: paddq %xmm4, %xmm0
+; X64-SSE4-NEXT: movdqa %xmm0, %xmm2
+; X64-SSE4-NEXT: psrlq $32, %xmm2
+; X64-SSE4-NEXT: pmuludq %xmm1, %xmm2
+; X64-SSE4-NEXT: movdqa %xmm1, %xmm3
+; X64-SSE4-NEXT: psrlq $32, %xmm3
+; X64-SSE4-NEXT: pmuludq %xmm0, %xmm3
+; X64-SSE4-NEXT: paddq %xmm2, %xmm3
+; X64-SSE4-NEXT: psllq $32, %xmm3
+; X64-SSE4-NEXT: pmuludq %xmm1, %xmm0
+; X64-SSE4-NEXT: paddq %xmm3, %xmm0
+; X64-SSE4-NEXT: pextrq $1, %xmm0, %rcx
+; X64-SSE4-NEXT: movq %xmm0, %rax
+; X64-SSE4-NEXT: imulq %rcx, %rax
+; X64-SSE4-NEXT: retq
+;
; X86-AVX1-LABEL: test_v16i64:
; X86-AVX1: # %bb.0:
; X86-AVX1-NEXT: pushl %ebp
@@ -1325,16 +1326,9 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X64-AVX1-NEXT: vpsllq $32, %xmm1, %xmm1
; X64-AVX1-NEXT: vpmuludq %xmm4, %xmm0, %xmm0
; X64-AVX1-NEXT: vpaddq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX1-NEXT: vpsrlq $32, %xmm0, %xmm2
-; X64-AVX1-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-AVX1-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X64-AVX1-NEXT: vpsllq $32, %xmm2, %xmm2
-; X64-AVX1-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX1-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX1-NEXT: vpextrq $1, %xmm0, %rcx
; X64-AVX1-NEXT: vmovq %xmm0, %rax
+; X64-AVX1-NEXT: imulq %rcx, %rax
; X64-AVX1-NEXT: vzeroupper
; X64-AVX1-NEXT: retq
;
@@ -1429,16 +1423,9 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X64-AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
; X64-AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; X64-AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-AVX2-NEXT: vpsrlq $32, %xmm0, %xmm2
-; X64-AVX2-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; X64-AVX2-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; X64-AVX2-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; X64-AVX2-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; X64-AVX2-NEXT: vpsllq $32, %xmm2, %xmm2
-; X64-AVX2-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; X64-AVX2-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; X64-AVX2-NEXT: vpextrq $1, %xmm0, %rcx
; X64-AVX2-NEXT: vmovq %xmm0, %rax
+; X64-AVX2-NEXT: imulq %rcx, %rax
; X64-AVX2-NEXT: vzeroupper
; X64-AVX2-NEXT: retq
;
@@ -1470,16 +1457,9 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; AVX512F-NEXT: vpsllq $32, %xmm2, %xmm2
; AVX512F-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; AVX512F-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512F-NEXT: vpsrlq $32, %xmm0, %xmm2
-; AVX512F-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; AVX512F-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512F-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX512F-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512F-NEXT: vpsllq $32, %xmm2, %xmm2
-; AVX512F-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512F-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; AVX512F-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512F-NEXT: vmovq %xmm0, %rax
+; AVX512F-NEXT: imulq %rcx, %rax
; AVX512F-NEXT: vzeroupper
; AVX512F-NEXT: retq
;
@@ -1511,16 +1491,9 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BW-NEXT: vpsrlq $32, %xmm0, %xmm2
-; AVX512BW-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; AVX512BW-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX512BW-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512BW-NEXT: vpsllq $32, %xmm2, %xmm2
-; AVX512BW-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BW-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; AVX512BW-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512BW-NEXT: vmovq %xmm0, %rax
+; AVX512BW-NEXT: imulq %rcx, %rax
; AVX512BW-NEXT: vzeroupper
; AVX512BW-NEXT: retq
;
@@ -1552,16 +1525,9 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; AVX512BWVL-NEXT: vpsrlq $32, %xmm0, %xmm2
-; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm2, %xmm2
-; AVX512BWVL-NEXT: vpsrldq {{.*#+}} xmm3 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero
-; AVX512BWVL-NEXT: vpmuludq %xmm3, %xmm0, %xmm3
-; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm3, %xmm2
-; AVX512BWVL-NEXT: vpsllq $32, %xmm2, %xmm2
-; AVX512BWVL-NEXT: vpmuludq %xmm1, %xmm0, %xmm0
-; AVX512BWVL-NEXT: vpaddq %xmm2, %xmm0, %xmm0
+; AVX512BWVL-NEXT: vpextrq $1, %xmm0, %rcx
; AVX512BWVL-NEXT: vmovq %xmm0, %rax
+; AVX512BWVL-NEXT: imulq %rcx, %rax
; AVX512BWVL-NEXT: vzeroupper
; AVX512BWVL-NEXT: retq
;
@@ -1601,9 +1567,10 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
define i32 @test_v2i32(<2 x i32> %a0) nounwind {
; SSE2-LABEL: test_v2i32:
; SSE2: # %bb.0:
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
-; SSE2-NEXT: pmuludq %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movd %xmm0, %ecx
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[1,1,1,1]
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: imull %ecx, %eax
; SSE2-NEXT: ret{{[l|q]}}
;
; SSE4-LABEL: test_v2i32:
@@ -1629,12 +1596,11 @@ define i32 @test_v4i32(<4 x i32> %a0) nounwind {
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[3,3,3,3]
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
; SSE2-NEXT: pmuludq %xmm1, %xmm2
+; SSE2-NEXT: movd %xmm2, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: pmuludq %xmm0, %xmm1
-; SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,0,0]
-; SSE2-NEXT: pmuludq %xmm1, %xmm0
-; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: imull %ecx, %eax
; SSE2-NEXT: ret{{[l|q]}}
;
; SSE4-LABEL: test_v4i32:
@@ -1663,20 +1629,18 @@ define i32 @test_v8i32(<8 x i32> %a0) nounwind {
; SSE2: # %bb.0:
; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,3,3]
; SSE2-NEXT: pmuludq %xmm1, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[1,1,3,3]
; SSE2-NEXT: pmuludq %xmm2, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm1[0,2,2,3]
-; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm0[2,3,2,3]
-; SSE2-NEXT: pmuludq %xmm0, %xmm2
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm2 = xmm2[2],xmm1[2],xmm2[3],xmm1[3]
+; SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSE2-NEXT: pmuludq %xmm2, %xmm0
+; SSE2-NEXT: movd %xmm0, %ecx
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,2,2,2]
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2]
; SSE2-NEXT: pmuludq %xmm0, %xmm1
-; SSE2-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm1[0],xmm2[1],xmm1[1]
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[0,0,0,0]
-; SSE2-NEXT: pmuludq %xmm2, %xmm0
-; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: imull %ecx, %eax
; SSE2-NEXT: ret{{[l|q]}}
;
; SSE4-LABEL: test_v8i32:
@@ -1751,21 +1715,19 @@ define i32 @test_v16i32(<16 x i32> %a0) nounwind {
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,2,2,3]
; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
; X86-SSE2-NEXT: pmuludq %xmm3, %xmm0
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2]
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
; X86-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE2-NEXT: pmuludq %xmm0, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
-; X86-SSE2-NEXT: pmuludq %xmm0, %xmm2
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,0,0]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; X86-SSE2-NEXT: pmuludq %xmm1, %xmm0
-; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: movd %xmm0, %ecx
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,0,2,2]
+; X86-SSE2-NEXT: pmuludq %xmm0, %xmm1
+; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: imull %ecx, %eax
; X86-SSE2-NEXT: movl %ebp, %esp
; X86-SSE2-NEXT: popl %ebp
; X86-SSE2-NEXT: retl
@@ -1787,21 +1749,19 @@ define i32 @test_v16i32(<16 x i32> %a0) nounwind {
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm2[0,2,2,3]
; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
; X64-SSE2-NEXT: pmuludq %xmm1, %xmm0
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,0,2,2]
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
; X64-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE2-NEXT: pmuludq %xmm0, %xmm1
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
-; X64-SSE2-NEXT: pmuludq %xmm0, %xmm2
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,0,0]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; X64-SSE2-NEXT: pmuludq %xmm1, %xmm0
-; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: movd %xmm0, %ecx
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,0,2,2]
+; X64-SSE2-NEXT: pmuludq %xmm0, %xmm1
+; X64-SSE2-NEXT: movd %xmm1, %eax
+; X64-SSE2-NEXT: imull %ecx, %eax
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v16i32:
@@ -1886,32 +1846,32 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X86-SSE2-NEXT: movl %esp, %ebp
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm3
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm4
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm6
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm6[1,1,3,3]
-; X86-SSE2-NEXT: pmuludq %xmm4, %xmm6
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[0,2,2,3]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm4[1,1,3,3]
-; X86-SSE2-NEXT: pmuludq %xmm4, %xmm5
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm5[0,2,2,3]
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm6 = xmm6[0],xmm4[0],xmm6[1],xmm4[1]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,3,3]
-; X86-SSE2-NEXT: pmuludq %xmm3, %xmm1
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm4
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm6
+; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm5
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm5[1,1,3,3]
+; X86-SSE2-NEXT: pmuludq %xmm6, %xmm5
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,2,2,3]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm6[1,1,3,3]
+; X86-SSE2-NEXT: pmuludq %xmm6, %xmm3
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm3[0,2,2,3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm6 = xmm1[1,1,3,3]
+; X86-SSE2-NEXT: pmuludq %xmm4, %xmm1
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm3[1,1,3,3]
-; X86-SSE2-NEXT: pmuludq %xmm4, %xmm7
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm7[0,2,2,3]
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm4[1,1,3,3]
+; X86-SSE2-NEXT: pmuludq %xmm6, %xmm7
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm4 = xmm7[0,2,2,3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm4[0],xmm1[1],xmm4[1]
; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm4
-; X86-SSE2-NEXT: pmuludq %xmm6, %xmm1
-; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm6
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm5[0,0,2,2]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm7[0,0,2,2]
; X86-SSE2-NEXT: pmuludq %xmm5, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3]
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm6
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,0,2,2]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm7[0,0,2,2]
+; X86-SSE2-NEXT: pmuludq %xmm5, %xmm3
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,2,2,3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm2[1,1,3,3]
; X86-SSE2-NEXT: pmuludq %xmm6, %xmm2
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[0,2,2,3]
@@ -1933,22 +1893,20 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X86-SSE2-NEXT: pmuludq %xmm2, %xmm4
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X86-SSE2-NEXT: pmuludq %xmm3, %xmm0
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2]
+; X86-SSE2-NEXT: pmuludq %xmm1, %xmm0
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,0,2,2]
; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,0,2,2]
; X86-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X86-SSE2-NEXT: pmuludq %xmm0, %xmm1
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
-; X86-SSE2-NEXT: pmuludq %xmm0, %xmm2
-; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,0,0]
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X86-SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X86-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; X86-SSE2-NEXT: pmuludq %xmm1, %xmm0
-; X86-SSE2-NEXT: movd %xmm0, %eax
+; X86-SSE2-NEXT: movd %xmm0, %ecx
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
+; X86-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,0,2,2]
+; X86-SSE2-NEXT: pmuludq %xmm0, %xmm1
+; X86-SSE2-NEXT: movd %xmm1, %eax
+; X86-SSE2-NEXT: imull %ecx, %eax
; X86-SSE2-NEXT: movl %ebp, %esp
; X86-SSE2-NEXT: popl %ebp
; X86-SSE2-NEXT: retl
@@ -1970,12 +1928,12 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm8 = xmm5[0,2,2,3]
; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm8[0],xmm1[1],xmm8[1]
; X64-SSE2-NEXT: pmuludq %xmm3, %xmm1
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm1[0,2,2,3]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm7 = xmm7[0,0,2,2]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm5[0,0,2,2]
-; X64-SSE2-NEXT: pmuludq %xmm7, %xmm1
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm1[0,2,2,3]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm3 = xmm5[0,0,2,2]
+; X64-SSE2-NEXT: pmuludq %xmm7, %xmm3
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm3[0,2,2,3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm5[0],xmm1[1],xmm5[1]
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm5 = xmm2[1,1,3,3]
; X64-SSE2-NEXT: pmuludq %xmm6, %xmm2
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
@@ -1997,22 +1955,20 @@ define i32 @test_v32i32(<32 x i32> %a0) nounwind {
; X64-SSE2-NEXT: pmuludq %xmm2, %xmm4
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,2,2,3]
; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
-; X64-SSE2-NEXT: pmuludq %xmm3, %xmm0
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,2,3]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm1[0,0,2,2]
+; X64-SSE2-NEXT: pmuludq %xmm1, %xmm0
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm3[0,0,2,2]
; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm4[0,0,2,2]
; X64-SSE2-NEXT: pmuludq %xmm1, %xmm2
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,2,2,3]
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; X64-SSE2-NEXT: pmuludq %xmm0, %xmm1
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm2 = xmm2[0,0,2,2]
-; X64-SSE2-NEXT: pmuludq %xmm0, %xmm2
-; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
-; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[0,0,0,0]
+; X64-SSE2-NEXT: movdqa %xmm0, %xmm1
+; X64-SSE2-NEXT: punpckhdq {{.*#+}} xmm1 = xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; X64-SSE2-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
; X64-SSE2-NEXT: pmuludq %xmm1, %xmm0
-; X64-SSE2-NEXT: movd %xmm0, %eax
+; X64-SSE2-NEXT: movd %xmm0, %ecx
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,2,2,2]
+; X64-SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm2[0,0,2,2]
+; X64-SSE2-NEXT: pmuludq %xmm0, %xmm1
+; X64-SSE2-NEXT: movd %xmm1, %eax
+; X64-SSE2-NEXT: imull %ecx, %eax
; X64-SSE2-NEXT: retq
;
; X86-SSE4-LABEL: test_v32i32:
More information about the llvm-commits
mailing list