[llvm] [X86] Add ISD::VECREDUCE_MUL lowering instead of matching in DAG (PR #207593)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Jul 5 08:39:23 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: Simon Pilgrim (RKSimon)
<details>
<summary>Changes</summary>
Add X86 support for ISD::VECREDUCE_MUL and remove it from combineArithReduction - most of the change is refactoring the custom promotion of vXi8 -> vXi16.
Another part of #<!-- -->194621
---
Patch is 65.25 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/207593.diff
5 Files Affected:
- (modified) llvm/lib/Target/X86/X86ISelLowering.cpp (+53-44)
- (modified) llvm/lib/Target/X86/X86TargetTransformInfo.cpp (+1)
- (modified) llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll (+3-3)
- (modified) llvm/test/CodeGen/X86/vector-reduce-mul.ll (+378-446)
- (modified) llvm/test/Transforms/PhaseOrdering/X86/vector-reductions-expanded.ll (+2-8)
``````````diff
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index b33b01f804c6b..8888070f9234a 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -1180,6 +1180,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
// SSE41 can use PHMINPOS to perform v16i8/v8i16 minmax reductions.
// Fallback to ReplaceNodeResults for vXi64 reductions on 32-bit targets.
for (auto VT : {MVT::v16i8, MVT::v8i16, MVT::v4i32, MVT::v2i64, MVT::i64}) {
+ setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
@@ -1571,6 +1572,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
@@ -2047,6 +2049,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
setOperationAction(ISD::VECREDUCE_AND, VT, Custom);
setOperationAction(ISD::VECREDUCE_OR, VT, Custom);
setOperationAction(ISD::VECREDUCE_XOR, VT, Custom);
+ setOperationAction(ISD::VECREDUCE_MUL, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMAX, VT, Custom);
setOperationAction(ISD::VECREDUCE_SMIN, VT, Custom);
setOperationAction(ISD::VECREDUCE_UMAX, VT, Custom);
@@ -2813,6 +2816,7 @@ X86TargetLowering::X86TargetLowering(const X86TargetMachine &TM,
ISD::ANY_EXTEND_VECTOR_INREG,
ISD::SIGN_EXTEND_VECTOR_INREG,
ISD::ZERO_EXTEND_VECTOR_INREG,
+ ISD::VECREDUCE_MUL,
ISD::SINT_TO_FP,
ISD::UINT_TO_FP,
ISD::FP_TO_SINT,
@@ -34351,6 +34355,7 @@ SDValue X86TargetLowering::LowerOperation(SDValue Op, SelectionDAG &DAG) const {
case ISD::VECREDUCE_AND:
case ISD::VECREDUCE_OR:
case ISD::VECREDUCE_XOR: return LowerVECREDUCE(Op, Subtarget, DAG, true);
+ case ISD::VECREDUCE_MUL: return LowerVECREDUCE(Op, Subtarget, DAG, false);
case ISD::FMINIMUM:
case ISD::FMAXIMUM:
case ISD::FMINIMUMNUM:
@@ -35153,6 +35158,12 @@ void X86TargetLowering::ReplaceNodeResults(SDNode *N,
}
return;
}
+ case ISD::VECREDUCE_MUL: {
+ assert(N->getValueType(0) == MVT::i64 && "Unexpected vector reduction");
+ if (SDValue Res = LowerVECREDUCE(SDValue(N, 0), Subtarget, DAG, false))
+ Results.push_back(Res);
+ return;
+ }
case ISD::VECREDUCE_SMAX:
case ISD::VECREDUCE_SMIN:
case ISD::VECREDUCE_UMAX:
@@ -47588,8 +47599,8 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
return SDValue();
ISD::NodeType Opc;
- SDValue Rdx = DAG.matchBinOpReduction(ExtElt, Opc,
- {ISD::ADD, ISD::MUL, ISD::FADD}, true);
+ SDValue Rdx =
+ DAG.matchBinOpReduction(ExtElt, Opc, {ISD::ADD, ISD::FADD}, true);
if (!Rdx)
return SDValue();
@@ -47606,10 +47617,10 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
unsigned NumElts = VecVT.getVectorNumElements();
unsigned EltSizeInBits = VecVT.getScalarSizeInBits();
- // Extend v4i8/v8i8 vector to v16i8, with undef upper 64-bits.
- auto WidenToV16I8 = [&](SDValue V, bool ZeroExtend) {
+ // ZeroExtend v4i8/v8i8 vector to v16i8, with undef upper 64-bits.
+ auto WidenToV16I8 = [&](SDValue V) {
if (V.getValueType() == MVT::v4i8) {
- if (ZeroExtend && Subtarget.hasSSE41()) {
+ if (Subtarget.hasSSE41()) {
V = DAG.getNode(ISD::INSERT_VECTOR_ELT, DL, MVT::v4i32,
DAG.getConstant(0, DL, MVT::v4i32),
DAG.getBitcast(MVT::i32, V),
@@ -47617,50 +47628,15 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
return DAG.getBitcast(MVT::v16i8, V);
}
V = DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v8i8, V,
- ZeroExtend ? DAG.getConstant(0, DL, MVT::v4i8)
- : DAG.getUNDEF(MVT::v4i8));
+ DAG.getConstant(0, DL, MVT::v4i8));
}
return DAG.getNode(ISD::CONCAT_VECTORS, DL, MVT::v16i8, V,
DAG.getUNDEF(MVT::v8i8));
};
- // vXi8 mul reduction - promote to vXi16 mul reduction.
- if (Opc == ISD::MUL) {
- if (VT != MVT::i8 || NumElts < 4 || !isPowerOf2_32(NumElts))
- return SDValue();
- if (VecVT.getSizeInBits() >= 128) {
- EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);
- SDValue Lo = getUnpackl(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
- SDValue Hi = getUnpackh(DAG, DL, VecVT, Rdx, DAG.getUNDEF(VecVT));
- Lo = DAG.getBitcast(WideVT, Lo);
- Hi = DAG.getBitcast(WideVT, Hi);
- Rdx = DAG.getNode(Opc, DL, WideVT, Lo, Hi);
- while (Rdx.getValueSizeInBits() > 128) {
- std::tie(Lo, Hi) = splitVector(Rdx, DAG, DL);
- Rdx = DAG.getNode(Opc, DL, Lo.getValueType(), Lo, Hi);
- }
- } else {
- Rdx = WidenToV16I8(Rdx, false);
- Rdx = getUnpackl(DAG, DL, MVT::v16i8, Rdx, DAG.getUNDEF(MVT::v16i8));
- Rdx = DAG.getBitcast(MVT::v8i16, Rdx);
- }
- if (NumElts >= 8)
- Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
- DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
- {4, 5, 6, 7, -1, -1, -1, -1}));
- Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
- DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
- {2, 3, -1, -1, -1, -1, -1, -1}));
- Rdx = DAG.getNode(Opc, DL, MVT::v8i16, Rdx,
- DAG.getVectorShuffle(MVT::v8i16, DL, Rdx, Rdx,
- {1, -1, -1, -1, -1, -1, -1, -1}));
- Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
- return DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, VT, Rdx, Index);
- }
-
// vXi8 add reduction - sub 128-bit vector.
if (VecVT == MVT::v4i8 || VecVT == MVT::v8i8) {
- Rdx = WidenToV16I8(Rdx, true);
+ Rdx = WidenToV16I8(Rdx);
Rdx = DAG.getNode(X86ISD::PSADBW, DL, MVT::v2i64, Rdx,
DAG.getConstant(0, DL, MVT::v16i8));
Rdx = DAG.getBitcast(MVT::v16i8, Rdx);
@@ -47706,7 +47682,7 @@ static SDValue combineArithReduction(SDNode *ExtElt, SelectionDAG &DAG,
EVT ByteVT = VecVT.changeVectorElementType(*DAG.getContext(), MVT::i8);
Rdx = DAG.getNode(ISD::TRUNCATE, DL, ByteVT, Rdx);
if (ByteVT.getSizeInBits() < 128)
- Rdx = WidenToV16I8(Rdx, true);
+ Rdx = WidenToV16I8(Rdx);
}
// Build the PSADBW, split as 128/256/512 bits for SSE/AVX2/AVX512BW.
@@ -47883,7 +47859,7 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
if (SDValue Cmp = combinePredicateReduction(N, DAG, Subtarget))
return Cmp;
- // Attempt to optimize ADD/FADD/MUL reductions with HADD, promotion etc..
+ // Attempt to optimize ADD/FADD reductions with HADD, promotion etc..
if (SDValue V = combineArithReduction(N, DAG, Subtarget))
return V;
@@ -47972,6 +47948,38 @@ static SDValue combineExtractVectorElt(SDNode *N, SelectionDAG &DAG,
return SDValue();
}
+static SDValue combineVECREDUCE_MUL(SDNode *N, SelectionDAG &DAG,
+ const X86Subtarget &Subtarget) {
+ SDValue Src = N->getOperand(0);
+ EVT SrcVT = Src.getValueType();
+ unsigned NumElts = SrcVT.getVectorNumElements();
+ SDLoc DL(N);
+
+ // vXi8 mul reduction - promote to vXi16 mul reduction.
+ if (!isPowerOf2_32(NumElts) || SrcVT.getScalarType() != MVT::i8)
+ return SDValue();
+
+ // Early out for v2i8 - no promotion necessary.
+ if (NumElts == 2) {
+ SDValue Hi = DAG.getVectorShuffle(SrcVT, DL, Src, Src, {1, -1});
+ SDValue Rdx = DAG.getNode(ISD::MUL, DL, SrcVT, Src, Hi);
+ return DAG.getExtractVectorElt(DL, N->getValueType(0), Rdx, 0);
+ }
+
+ if (SrcVT.getSizeInBits() >= 128) {
+ EVT WideVT = EVT::getVectorVT(*DAG.getContext(), MVT::i16, NumElts / 2);
+ SDValue Lo = getUnpackl(DAG, DL, SrcVT, Src, DAG.getUNDEF(SrcVT));
+ SDValue Hi = getUnpackh(DAG, DL, SrcVT, Src, DAG.getUNDEF(SrcVT));
+ Src = DAG.getNode(ISD::MUL, DL, WideVT, DAG.getBitcast(WideVT, Lo),
+ DAG.getBitcast(WideVT, Hi));
+ } else {
+ EVT ExtVT = SrcVT.changeVectorElementType(*DAG.getContext(), MVT::i16);
+ Src = DAG.getNode(ISD::ANY_EXTEND, DL, ExtVT, Src);
+ }
+ Src = DAG.getNode(ISD::VECREDUCE_MUL, DL, MVT::i16, Src);
+ return DAG.getZExtOrTrunc(Src, DL, N->getValueType(0));
+}
+
// Convert (vXiY *ext(vXi1 bitcast(iX))) to extend_in_reg(broadcast(iX)).
// This is more or less the reverse of combineBitcastvxi1.
static SDValue combineToExtendBoolVectorInReg(
@@ -62931,6 +62939,7 @@ SDValue X86TargetLowering::PerformDAGCombine(SDNode *N,
case X86ISD::VFCMULC:
case X86ISD::VFMULC: return combineFMulcFCMulc(N, DAG, Subtarget);
case ISD::FNEG: return combineFneg(N, DAG, DCI, Subtarget);
+ case ISD::VECREDUCE_MUL: return combineVECREDUCE_MUL(N, DAG, Subtarget);
case ISD::TRUNCATE: return combineTruncate(N, DAG, Subtarget);
case X86ISD::VTRUNC: return combineVTRUNC(N, DAG, DCI);
case X86ISD::VTRUNCS:
diff --git a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
index 2cb99d0c2090a..2993173d35edc 100644
--- a/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
+++ b/llvm/lib/Target/X86/X86TargetTransformInfo.cpp
@@ -6851,6 +6851,7 @@ bool X86TTIImpl::shouldExpandReduction(const IntrinsicInst *II) const {
switch (II->getIntrinsicID()) {
default:
return true;
+ case Intrinsic::vector_reduce_mul:
case Intrinsic::vector_reduce_smax:
case Intrinsic::vector_reduce_smin:
case Intrinsic::vector_reduce_umax:
diff --git a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
index 1e14256db3584..ddd94bfd3e4b6 100644
--- a/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
+++ b/llvm/test/CodeGen/X86/avx512-intrinsics-fast-isel.ll
@@ -6895,7 +6895,7 @@ define i32 @test_mm512_reduce_mul_epi32(<8 x i64> %__W) nounwind {
; CHECK-LABEL: test_mm512_reduce_mul_epi32:
; CHECK: # %bb.0: # %entry
; CHECK-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; CHECK-NEXT: vpmulld %zmm1, %zmm0, %zmm0
+; CHECK-NEXT: vpmulld %ymm1, %ymm0, %ymm0
; CHECK-NEXT: vextracti128 $1, %ymm0, %xmm1
; CHECK-NEXT: vpmulld %xmm1, %xmm0, %xmm0
; CHECK-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -7000,7 +7000,7 @@ define i32 @test_mm512_mask_reduce_mul_epi32(i16 zeroext %__M, <8 x i64> %__W) n
; X86-NEXT: vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; X86-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
; X86-NEXT: vextracti64x4 $1, %zmm1, %ymm0
-; X86-NEXT: vpmulld %zmm0, %zmm1, %zmm0
+; X86-NEXT: vpmulld %ymm0, %ymm1, %ymm0
; X86-NEXT: vextracti128 $1, %ymm0, %xmm1
; X86-NEXT: vpmulld %xmm1, %xmm0, %xmm0
; X86-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -7017,7 +7017,7 @@ define i32 @test_mm512_mask_reduce_mul_epi32(i16 zeroext %__M, <8 x i64> %__W) n
; X64-NEXT: vpbroadcastd {{.*#+}} zmm1 = [1,1,1,1,1,1,1,1,1,1,1,1,1,1,1,1]
; X64-NEXT: vmovdqa32 %zmm0, %zmm1 {%k1}
; X64-NEXT: vextracti64x4 $1, %zmm1, %ymm0
-; X64-NEXT: vpmulld %zmm0, %zmm1, %zmm0
+; X64-NEXT: vpmulld %ymm0, %ymm1, %ymm0
; X64-NEXT: vextracti128 $1, %ymm0, %xmm1
; X64-NEXT: vpmulld %xmm1, %xmm0, %xmm0
; X64-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
diff --git a/llvm/test/CodeGen/X86/vector-reduce-mul.ll b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
index 103c3329ede5c..b1d5b188b8de2 100644
--- a/llvm/test/CodeGen/X86/vector-reduce-mul.ll
+++ b/llvm/test/CodeGen/X86/vector-reduce-mul.ll
@@ -884,7 +884,7 @@ define i64 @test_v8i64(<8 x i64> %a0) nounwind {
; AVX512DQVL-LABEL: test_v8i64:
; AVX512DQVL: # %bb.0:
; AVX512DQVL-NEXT: vextracti64x4 $1, %zmm0, %ymm1
-; AVX512DQVL-NEXT: vpmullq %zmm1, %zmm0, %zmm0
+; AVX512DQVL-NEXT: vpmullq %ymm1, %ymm0, %ymm0
; AVX512DQVL-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX512DQVL-NEXT: vpmullq %xmm1, %xmm0, %xmm0
; AVX512DQVL-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -903,50 +903,28 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: movl %esp, %ebp
; X86-SSE2-NEXT: andl $-16, %esp
; X86-SSE2-NEXT: subl $16, %esp
-; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm4
-; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm3
-; X86-SSE2-NEXT: movdqa %xmm2, %xmm5
-; X86-SSE2-NEXT: psrlq $32, %xmm5
-; X86-SSE2-NEXT: pmuludq %xmm3, %xmm5
-; X86-SSE2-NEXT: movdqa %xmm3, %xmm6
-; X86-SSE2-NEXT: psrlq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm2, %xmm6
-; X86-SSE2-NEXT: paddq %xmm5, %xmm6
-; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm5
-; X86-SSE2-NEXT: psllq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm3, %xmm2
-; X86-SSE2-NEXT: paddq %xmm6, %xmm2
-; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
-; X86-SSE2-NEXT: psrlq $32, %xmm3
-; X86-SSE2-NEXT: pmuludq %xmm5, %xmm3
+; X86-SSE2-NEXT: movdqa 72(%ebp), %xmm5
+; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm4
+; X86-SSE2-NEXT: psrlq $32, %xmm4
+; X86-SSE2-NEXT: pmuludq %xmm5, %xmm4
; X86-SSE2-NEXT: movdqa %xmm5, %xmm6
; X86-SSE2-NEXT: psrlq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm0, %xmm6
-; X86-SSE2-NEXT: paddq %xmm3, %xmm6
-; X86-SSE2-NEXT: movdqa 8(%ebp), %xmm3
+; X86-SSE2-NEXT: pmuludq %xmm3, %xmm6
+; X86-SSE2-NEXT: paddq %xmm4, %xmm6
+; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm4
; X86-SSE2-NEXT: psllq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm5, %xmm0
-; X86-SSE2-NEXT: paddq %xmm6, %xmm0
-; X86-SSE2-NEXT: movdqa %xmm3, %xmm5
+; X86-SSE2-NEXT: pmuludq %xmm5, %xmm3
+; X86-SSE2-NEXT: paddq %xmm6, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm1, %xmm5
; X86-SSE2-NEXT: psrlq $32, %xmm5
; X86-SSE2-NEXT: pmuludq %xmm4, %xmm5
; X86-SSE2-NEXT: movdqa %xmm4, %xmm6
; X86-SSE2-NEXT: psrlq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm3, %xmm6
-; X86-SSE2-NEXT: paddq %xmm5, %xmm6
-; X86-SSE2-NEXT: movdqa 40(%ebp), %xmm5
-; X86-SSE2-NEXT: psllq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm4, %xmm3
-; X86-SSE2-NEXT: paddq %xmm6, %xmm3
-; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
-; X86-SSE2-NEXT: psrlq $32, %xmm4
-; X86-SSE2-NEXT: pmuludq %xmm5, %xmm4
-; X86-SSE2-NEXT: movdqa %xmm5, %xmm6
-; X86-SSE2-NEXT: psrlq $32, %xmm6
; X86-SSE2-NEXT: pmuludq %xmm1, %xmm6
-; X86-SSE2-NEXT: paddq %xmm4, %xmm6
+; X86-SSE2-NEXT: paddq %xmm5, %xmm6
; X86-SSE2-NEXT: psllq $32, %xmm6
-; X86-SSE2-NEXT: pmuludq %xmm5, %xmm1
+; X86-SSE2-NEXT: pmuludq %xmm4, %xmm1
; X86-SSE2-NEXT: paddq %xmm6, %xmm1
; X86-SSE2-NEXT: movdqa %xmm1, %xmm4
; X86-SSE2-NEXT: psrlq $32, %xmm4
@@ -955,9 +933,31 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
; X86-SSE2-NEXT: psrlq $32, %xmm5
; X86-SSE2-NEXT: pmuludq %xmm1, %xmm5
; X86-SSE2-NEXT: paddq %xmm4, %xmm5
+; X86-SSE2-NEXT: movdqa 56(%ebp), %xmm4
; X86-SSE2-NEXT: psllq $32, %xmm5
; X86-SSE2-NEXT: pmuludq %xmm3, %xmm1
; X86-SSE2-NEXT: paddq %xmm5, %xmm1
+; X86-SSE2-NEXT: movdqa %xmm2, %xmm3
+; X86-SSE2-NEXT: psrlq $32, %xmm3
+; X86-SSE2-NEXT: pmuludq %xmm4, %xmm3
+; X86-SSE2-NEXT: movdqa %xmm4, %xmm5
+; X86-SSE2-NEXT: psrlq $32, %xmm5
+; X86-SSE2-NEXT: pmuludq %xmm2, %xmm5
+; X86-SSE2-NEXT: paddq %xmm3, %xmm5
+; X86-SSE2-NEXT: movdqa 24(%ebp), %xmm3
+; X86-SSE2-NEXT: psllq $32, %xmm5
+; X86-SSE2-NEXT: pmuludq %xmm4, %xmm2
+; X86-SSE2-NEXT: paddq %xmm5, %xmm2
+; X86-SSE2-NEXT: movdqa %xmm0, %xmm4
+; X86-SSE2-NEXT: psrlq $32, %xmm4
+; X86-SSE2-NEXT: pmuludq %xmm3, %xmm4
+; X86-SSE2-NEXT: movdqa %xmm3, %xmm5
+; X86-SSE2-NEXT: psrlq $32, %xmm5
+; X86-SSE2-NEXT: pmuludq %xmm0, %xmm5
+; X86-SSE2-NEXT: paddq %xmm4, %xmm5
+; X86-SSE2-NEXT: psllq $32, %xmm5
+; X86-SSE2-NEXT: pmuludq %xmm3, %xmm0
+; X86-SSE2-NEXT: paddq %xmm5, %xmm0
; X86-SSE2-NEXT: movdqa %xmm0, %xmm3
; X86-SSE2-NEXT: psrlq $32, %xmm3
; X86-SSE2-NEXT: pmuludq %xmm2, %xmm3
@@ -998,56 +998,56 @@ define i64 @test_v16i64(<16 x i64> %a0) nounwind {
;
; X64-SSE-LABEL: test_v16i64:
; X64-SSE: # %bb.0:
-; X64-SSE-NEXT: movdqa %xmm2, %xmm8
+; X64-SSE-NEXT: movdqa %xmm3, %xmm8
; X64-SSE-NEXT: psrlq $32, %xmm8
-; X64-SSE-NEXT: pmuludq %xmm6, %xmm8
-; X64-SSE-NEXT: movdqa %xmm6, %xmm9
+; X64-SSE-NEXT: pmuludq %xmm7, %xmm8
+; X64-SSE-NEXT: movdqa %xmm7, %xmm9
; X64-SSE-NEXT: psrlq $32, %xmm9
-; X64-SSE-NEXT: pmuludq %xmm2, %xmm9
+; X64-SSE-NEXT: pmuludq %xmm3, %xmm9
; X64-SSE-NEXT: paddq %xmm8, %xmm9
; X64-SSE-NEXT: psllq $32, %xmm9
-; X64-SSE-NEXT: pmuludq %xmm6, %xmm2
-; X64-SSE-NEXT: paddq %xmm9, %xmm2
-; X64-SSE-NEXT: movdqa %xmm0, %xmm6
-; X64-SSE-NEXT: psrlq $32, %xmm6
-; X64-SSE-NEXT: pmuludq %xmm4, %xmm6
-; X64-SSE-NEXT: movdqa %xmm4, %xmm8
+; X64-SSE-NEXT: pmuludq %xmm7, %xmm3
+; X64-SSE-NEXT: paddq %xmm9, %xmm3
+; X64-SSE-NEXT: movdqa %xmm1, %xmm7
+; X64-SSE-NEXT: psrlq $32, %xmm7
+; X64-SSE-NEXT: pmuludq %xmm5, %xmm7
+; X64-SSE-NEXT: movdqa %xmm5, %xmm8
; X64-SSE-NEXT: psrlq $32, %xmm8
-; X64-SSE-NEXT: pmuludq %xmm0, %xmm8
-; X64-SSE-NEXT: paddq %xmm6, %xmm8
+; X64-SSE-NEXT: pmuludq %xmm1, %xmm8
+; X64-SSE-NEXT: paddq %xmm7, %xmm8
; X64-SSE-NEXT: psllq $32, %xmm8
-; X64-SSE-NEXT: pmuludq %xmm4, %xmm0
-; X64-SSE-NEXT: paddq %xmm8, %xmm0
-; X64-SSE-NEXT: movdqa %xmm3, %xmm4
-; X64-SSE-NEXT: psrlq $32, %xmm4
-; X64-SSE-NEXT: pmuludq %xmm7, %xmm4
-; X64-SSE-NEXT: movdqa %xmm7, %xmm6
-; X64-SSE-NEXT: psrlq $32, %xmm6
-; X64-SSE-NEXT: pmuludq %xmm3, %xmm6
-; X64-SSE-NEXT: paddq %xmm4, %xmm6
-; X64-SSE-NEXT: psllq $32, %xmm6
-; X64-SSE-NEXT: pmuludq %xmm7, %xmm3
-; X64-SSE-NEXT: paddq %xmm6, %xmm3
-; X64-SSE-NEXT: movdqa %xmm1, %xmm4
-; X64-SSE-NEXT: psrlq $32, %xmm4
-; X64-SSE-NEXT: pmuludq %xmm5, %xmm4
-; X64-SSE-NEXT: movdqa %xmm5, %xmm6
-; X64-SSE-NEXT: psrlq $32, %xmm6
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm6
-; X64-SSE-NEXT: paddq %xmm4, %xmm6
-; X64-SSE-NEXT: psllq $32, %xmm6
; X64-SSE-NEXT: pmuludq %xmm5, %xmm1
-; X64-SSE-NEXT: paddq %xmm6, %xmm1
-; X64-SSE-NEXT: movdqa %xmm1, %xmm4
-; X64-SSE-NEXT: psrlq $32, %xmm4
-; X64-SSE-NEXT: pmuludq %xmm3, %xmm4
-; X64-SSE-NEXT: movdqa %xmm3, %xmm5
+; X64-SSE-NEXT: paddq %xmm8, %xmm1
+; X64-SSE-NEXT: movdqa %xmm1, %xmm5
; X64-SSE-NEXT: psrlq $32, %xmm5
-; X64-SSE-NEXT: pmuludq %xmm1, %xmm5
-; X64-SSE-NEXT: paddq %xmm4, %xmm5
-; X64-SSE-NEXT: psllq $32, %xmm5
+; X64-SSE-NEXT: pmuludq %xmm3, %xmm5
+; X64-SSE-NEXT: movdqa %xmm3, %xmm7
+; X64-SSE-NEXT: psrlq $32, %xmm7
+; X64-SSE-NEXT: pmuludq %xmm1, %xmm7
+; X64-SSE-NEXT: paddq %xmm5, %xmm7
+; X64-SSE-NEXT: psllq $32, %xmm7
; X64-SSE-NEXT: pmuludq %xmm3, %xmm1
-; X64-SSE-NEXT: paddq %xmm5, %xmm1
+; X64-SSE-NEXT: paddq %xmm7, %xmm1
+; X64-SSE-NEXT: movdqa %xmm2, %xmm3
+; X64-SSE-NEXT: psrlq $32, %xmm3
+; X64-SSE-NEXT: pmuludq %xmm6, %xmm3
+; X64-SSE-NEXT: movdqa %xmm6, %xmm5
+; X64-SSE-NEXT: psrlq $32, %xmm5
+; X64-SSE-NEXT: pmuludq %xmm2, %xmm5
+; X64-SSE-NEXT: paddq %xmm3, %xmm5
+; X64-SSE-NEXT: psllq $32, %xmm5
+; X64-SSE-NEXT: pmuludq %xmm6, %xmm2
+; X64-SSE-NEXT: paddq %xmm5, %xmm2
+; X64-SSE-NEXT: movdqa %xmm0, %xmm3
+; X64-SSE-NEXT: psrlq $32, %xmm3
+; X64-SSE-NEXT: pmuludq %xmm4, %xmm3
+; X64-SSE-NEXT: movdqa %xmm4, %xmm5
+; X64-SSE-NEXT: psrlq $32, %xmm5
+; X64-SSE-NEXT: pmuludq %xmm0, %xmm5
+; X64-SSE-NEXT: paddq %xmm3, %xmm5
+; X64-SSE-NEXT: psllq $32, %xmm5
+; X64-SSE-NEXT: pmuludq %xmm4, %xmm0
+; X64-SSE-NEXT: paddq %xmm5, %xmm0
; X64-SSE-NEXT: movdqa %xmm0, ...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/207593
More information about the llvm-commits
mailing list