[llvm] [X86] combineAddOfPMADDWD - add PMADDUBSW support (PR #228359)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 2 01:20:52 PDT 2026
llvmorg-github-actions[bot] wrote:
<!--LLVM PR SUMMARY COMMENT-->
@llvm/pr-subscribers-backend-x86
Author: Jeong Jihyeon (JihyeonJeong129)
<details>
<summary>Changes</summary>
This patch extends the ADD-of-PMADDWD DAG combine to support PMADDUBSW for 128, 256, and 512-bit vectors and renames it to `combineAddOfPMADD`.
Existing PMADDWD behavior is preserved. The PMADDUBSW path only combines when signed i16 overflow is ruled out and both operand interleaves reconstruct existing inputs without additional shuffles. The shuffle restriction is motivated by #<!-- -->208457
Tests cover:
- Safe inputs, including unsigned i8 values with the high bit set
- Commuted ADD operands
- Positive signed i16 overflow
- Negative signed i16 overflow
- Independent inputs requiring additional shuffles
Fixes #<!-- -->223367
Assisted-by: Codex
---
Patch is 54.44 KiB, truncated to 20.00 KiB below, full version: https://github.com/llvm/llvm-project/pull/228359.diff
4 Files Affected:
- (modified) llvm/lib/Target/X86/X86ISelLowering.cpp (+70-11)
- (added) llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll (+173)
- (added) llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll (+120)
- (modified) llvm/test/CodeGen/X86/combine-pmaddubsw.ll (+245-3)
``````````diff
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 5a7235d7249a3ff..1f5d641d8d2597e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -59575,17 +59575,52 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N,
PMADDBuilder);
}
-// ADD(VPMADDWD(X,Y),VPMADDWD(Z,W)) -> VPMADDWD(SHUFFLE(X,Z), SHUFFLE(Y,W))
-// If upper element in each pair of both VPMADDWD are zero then we can merge
-// the operand elements and use the implicit add of VPMADDWD.
-// TODO: Add support for VPMADDUBSW (which isn't commutable).
-static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
- const SDLoc &DL, EVT VT) {
- if (N0.getOpcode() != N1.getOpcode() || N0.getOpcode() != X86ISD::VPMADDWD)
+// Return an existing input if interleaving the even elements of EvenOp and
+// OddOp reconstructs it without any additional shuffling.
+static SDValue getInterleavedPMADDInput(SelectionDAG &DAG, SDValue EvenOp,
+ SDValue OddOp) {
+ MVT VT = EvenOp.getSimpleValueType();
+ unsigned NumElts = VT.getVectorNumElements();
+ SDValue Src;
+ for (unsigned I = 0; I != 2; ++I) {
+ SDValue Op = peekThroughBitcasts(I == 0 ? EvenOp : OddOp);
+ SmallVector<SDValue, 2> Inputs;
+ SmallVector<int, 64> Mask, ScaledMask;
+ if (!getTargetShuffleInputs(Op, Inputs, Mask, DAG)) {
+ Inputs.assign(1, Op);
+ Mask.resize(NumElts);
+ std::iota(Mask.begin(), Mask.end(), 0);
+ }
+ if (!scaleShuffleMaskElts(NumElts, Mask, ScaledMask))
+ return SDValue();
+
+ for (unsigned J = 0; J != NumElts; J += 2) {
+ int M = ScaledMask[J];
+ if (M < 0 || unsigned(M) % NumElts != J + I)
+ return SDValue();
+ SDValue Input = peekThroughBitcasts(Inputs[M / NumElts]);
+ if (!Input.getValueType().isVector() ||
+ Input.getValueSizeInBits() != VT.getSizeInBits() ||
+ (Src && Src != Input))
+ return SDValue();
+ Src = Input;
+ }
+ }
+ return Src;
+}
+
+// ADD(PMADD(X,Y),PMADD(Z,W)) -> PMADD(SHUFFLE(X,Z), SHUFFLE(Y,W))
+// If the upper product in each pair of both PMADDs is zero, merge the operand
+// elements and use the implicit add of PMADD.
+static SDValue combineAddOfPMADD(SelectionDAG &DAG, SDValue N0, SDValue N1,
+ const SDLoc &DL, EVT VT) {
+ if (N0.getOpcode() != N1.getOpcode() ||
+ (N0.getOpcode() != X86ISD::VPMADDWD &&
+ N0.getOpcode() != X86ISD::VPMADDUBSW))
return SDValue();
// TODO: Add 256/512-bit support once VPMADDWD combines with shuffles.
- if (VT.getSizeInBits() > 128)
+ if (N0.getOpcode() == X86ISD::VPMADDWD && VT.getSizeInBits() > 128)
return SDValue();
unsigned NumElts = VT.getVectorNumElements();
@@ -59602,7 +59637,31 @@ static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
if (!Op0HiZero || !Op1HiZero)
return SDValue();
- // Create a shuffle mask packing the lower elements from each VPMADDWD.
+ // Each unsigned i8 * signed i8 product fits in signed i16. Ensure their sum
+ // does too, so that the saturation in VPMADDUBSW does not change the result.
+ if (N0.getOpcode() == X86ISD::VPMADDUBSW &&
+ !DAG.willNotOverflowAdd(/*IsSigned=*/true, N0, N1))
+ return SDValue();
+
+ // Only combine PMADDUBSW when both interleaves reconstruct existing inputs;
+ // introducing new shuffles can make this combine unprofitable.
+ if (N0.getOpcode() == X86ISD::VPMADDUBSW) {
+ // Try both ADD operand orders, keeping the unsigned/signed operand order
+ // of each PMADDUBSW and matching both inputs in the same direction.
+ for (unsigned I = 0; I != 2; ++I) {
+ SDValue LHS =
+ getInterleavedPMADDInput(DAG, N0.getOperand(0), N1.getOperand(0));
+ SDValue RHS =
+ getInterleavedPMADDInput(DAG, N0.getOperand(1), N1.getOperand(1));
+ if (LHS && RHS)
+ return DAG.getNode(N0.getOpcode(), DL, VT, DAG.getBitcast(OpVT, LHS),
+ DAG.getBitcast(OpVT, RHS));
+ std::swap(N0, N1);
+ }
+ return SDValue();
+ }
+
+ // Create a shuffle mask packing the lower elements from each PMADD.
SmallVector<int> Mask;
for (int i = 0; i != (int)NumElts; ++i) {
Mask.push_back(2 * i);
@@ -59613,7 +59672,7 @@ static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
DAG.getVectorShuffle(OpVT, DL, N0.getOperand(0), N1.getOperand(0), Mask);
SDValue RHS =
DAG.getVectorShuffle(OpVT, DL, N0.getOperand(1), N1.getOperand(1), Mask);
- return DAG.getNode(X86ISD::VPMADDWD, DL, VT, LHS, RHS);
+ return DAG.getNode(N0.getOpcode(), DL, VT, LHS, RHS);
}
/// CMOV of constants requires materializing constant operands in registers.
@@ -59742,7 +59801,7 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
return MAdd;
if (SDValue MAdd = matchPMADDWD_2(DAG, N, DL, VT, Subtarget))
return MAdd;
- if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT))
+ if (SDValue MAdd = combineAddOfPMADD(DAG, Op0, Op1, DL, VT))
return MAdd;
// Try to synthesize horizontal adds from adds of shuffles.
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
new file mode 100644
index 000000000000000..37a4f475ebaba9a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
@@ -0,0 +1,173 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefix=AVX512
+
+; 255 * 63 + 255 * 63 fits in signed i16.
+; The unsigned input may have its high bit set; keep it as operand 0.
+define <16 x i16> @combine_pmaddubsw_add_u8_s6(<32 x i8> %a0, <32 x i8> %a1) {
+; AVX2-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: retq
+ %and = and <32 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %even1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %odd0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %odd1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %even = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %even0, <32 x i8> %even1)
+ %odd = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %odd0, <32 x i8> %odd1)
+ %res = add <16 x i16> %even, %odd
+ ret <16 x i16> %res
+}
+
+; The ADD operands can be commuted without changing PMADDUBSW operand roles.
+define <16 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<32 x i8> %a0, <32 x i8> %a1) {
+; AVX2-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: retq
+ %and = and <32 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %even1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %odd0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %odd1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %even = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %even0, <32 x i8> %even1)
+ %odd = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %odd0, <32 x i8> %odd1)
+ %res = add <16 x i16> %odd, %even
+ ret <16 x i16> %res
+}
+
+; Only the last output lane can overflow: 255 * 127 + 255 * 127 wraps to
+; -766, but a combined PMADDUBSW saturates to 32767. All other lanes are
+; bounded by 127 * 127 + 127 * 127.
+define <16 x i16> @combine_pmaddubsw_add_positive_overflow(<32 x i8> %a0, <32 x i8> %a1) {
+; AVX2-LABEL: combine_pmaddubsw_add_positive_overflow:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_positive_overflow:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX512-NEXT: retq
+ %and0 = and <32 x i8> %a0, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 -1, i8 -1>
+ %and1 = and <32 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %even0 = shufflevector <32 x i8> %and0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %even1 = shufflevector <32 x i8> %and1, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %odd0 = shufflevector <32 x i8> %and0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %odd1 = shufflevector <32 x i8> %and1, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %even = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %even0, <32 x i8> %even1)
+ %odd = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %odd0, <32 x i8> %odd1)
+ %res = add <16 x i16> %even, %odd
+ ret <16 x i16> %res
+}
+
+; 255 * -128 + 255 * -128 wraps to 256, but a combined PMADDUBSW saturates to -32768.
+define <16 x i16> @combine_pmaddubsw_add_negative_overflow(<32 x i8> %a0, <32 x i8> %a1) {
+; AVX2-LABEL: combine_pmaddubsw_add_negative_overflow:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_negative_overflow:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX512-NEXT: retq
+ %or = or <32 x i8> %a1, <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>
+ %even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %even1 = shufflevector <32 x i8> %or, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %odd0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %odd1 = shufflevector <32 x i8> %or, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %even = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %even0, <32 x i8> %even1)
+ %odd = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %odd0, <32 x i8> %odd1)
+ %res = add <16 x i16> %even, %odd
+ ret <16 x i16> %res
+}
+
+; Independent inputs require new shuffles. Preserve the two PMADDUBSWs
+; and the ADD instead of introducing those shuffles.
+define <16 x i16> @combine_pmaddubsw_add_independent_inputs(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %a2, <32 x i8> %a3) {
+; AVX2-LABEL: combine_pmaddubsw_add_independent_inputs:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX2-NEXT: vpand %ymm5, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm1
+; AVX2-NEXT: vpand %ymm5, %ymm3, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_independent_inputs:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; AVX512-NEXT: vpand %ymm4, %ymm0, %ymm0
+; AVX512-NEXT: vpbroadcastb {{.*#+}} ymm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512-NEXT: vpand %ymm5, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpand %ymm4, %ymm2, %ymm1
+; AVX512-NEXT: vpand %ymm5, %ymm3, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm1, %ymm1
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: retq
+ %and0 = and <32 x i8> %a0, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and1 = and <32 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %and2 = and <32 x i8> %a2, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and3 = and <32 x i8> %a3, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127...
[truncated]
``````````
</details>
https://github.com/llvm/llvm-project/pull/228359
More information about the llvm-commits
mailing list