[llvm] [X86] combineAddOfPMADDWD - add PMADDUBSW support (PR #228359)
Jeong Jihyeon via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 2 01:19:43 PDT 2026
https://github.com/JihyeonJeong129 created https://github.com/llvm/llvm-project/pull/228359
This patch extends the ADD-of-PMADDWD DAG combine to support PMADDUBSW for 128, 256, and 512-bit vectors and renames it to `combineAddOfPMADD`.
Existing PMADDWD behavior is preserved. The PMADDUBSW path only combines when signed i16 overflow is ruled out and both operand interleaves reconstruct existing inputs without additional shuffles. The shuffle restriction is motivated by #208457
Tests cover:
- Safe inputs, including unsigned i8 values with the high bit set
- Commuted ADD operands
- Positive signed i16 overflow
- Negative signed i16 overflow
- Independent inputs requiring additional shuffles
Fixes #223367
Assisted-by: Codex
>From 06498797aebf2c91e72f1a49d01cb0a99fe781ea Mon Sep 17 00:00:00 2001
From: Jihyeon Jeong <jh.jeong129 at gmail.com>
Date: Fri, 2 Oct 2026 04:11:41 +0000
Subject: [PATCH 1/2] [X86] Add tests for add-of-PMADDUBSW combines
---
.../test/CodeGen/X86/combine-pmaddubsw-256.ll | 201 +++++++++++
.../test/CodeGen/X86/combine-pmaddubsw-512.ll | 134 +++++++
llvm/test/CodeGen/X86/combine-pmaddubsw.ll | 332 +++++++++++++++++-
3 files changed, 664 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
create mode 100644 llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
new file mode 100644
index 000000000000000..082b3b60bb91afc
--- /dev/null
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
@@ -0,0 +1,201 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefix=AVX512
+
+; 255 * 63 + 255 * 63 fits in signed i16.
+; The unsigned input may have its high bit set; keep it as operand 0.
+define <16 x i16> @combine_pmaddubsw_add_u8_s6(<32 x i8> %a0, <32 x i8> %a1) {
+; AVX2-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX512-NEXT: retq
+ %and = and <32 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %even1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %odd0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %odd1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %even = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %even0, <32 x i8> %even1)
+ %odd = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %odd0, <32 x i8> %odd1)
+ %res = add <16 x i16> %even, %odd
+ ret <16 x i16> %res
+}
+
+; The ADD operands can be commuted without changing PMADDUBSW operand roles.
+define <16 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<32 x i8> %a0, <32 x i8> %a1) {
+; AVX2-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddw %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: retq
+ %and = and <32 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %even1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %odd0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %odd1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %even = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %even0, <32 x i8> %even1)
+ %odd = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %odd0, <32 x i8> %odd1)
+ %res = add <16 x i16> %odd, %even
+ ret <16 x i16> %res
+}
+
+; Only the last output lane can overflow: 255 * 127 + 255 * 127 wraps to
+; -766, but a combined PMADDUBSW saturates to 32767. All other lanes are
+; bounded by 127 * 127 + 127 * 127.
+define <16 x i16> @combine_pmaddubsw_add_positive_overflow(<32 x i8> %a0, <32 x i8> %a1) {
+; AVX2-LABEL: combine_pmaddubsw_add_positive_overflow:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_positive_overflow:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX512-NEXT: retq
+ %and0 = and <32 x i8> %a0, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 -1, i8 -1>
+ %and1 = and <32 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %even0 = shufflevector <32 x i8> %and0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %even1 = shufflevector <32 x i8> %and1, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %odd0 = shufflevector <32 x i8> %and0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %odd1 = shufflevector <32 x i8> %and1, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %even = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %even0, <32 x i8> %even1)
+ %odd = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %odd0, <32 x i8> %odd1)
+ %res = add <16 x i16> %even, %odd
+ ret <16 x i16> %res
+}
+
+; 255 * -128 + 255 * -128 wraps to 256, but a combined PMADDUBSW saturates to -32768.
+define <16 x i16> @combine_pmaddubsw_add_negative_overflow(<32 x i8> %a0, <32 x i8> %a1) {
+; AVX2-LABEL: combine_pmaddubsw_add_negative_overflow:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_negative_overflow:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX512-NEXT: retq
+ %or = or <32 x i8> %a1, <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>
+ %even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %even1 = shufflevector <32 x i8> %or, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %odd0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %odd1 = shufflevector <32 x i8> %or, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %even = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %even0, <32 x i8> %even1)
+ %odd = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %odd0, <32 x i8> %odd1)
+ %res = add <16 x i16> %even, %odd
+ ret <16 x i16> %res
+}
+
+; Independent inputs require new shuffles. Preserve the two PMADDUBSWs
+; and the ADD instead of introducing those shuffles.
+define <16 x i16> @combine_pmaddubsw_add_independent_inputs(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %a2, <32 x i8> %a3) {
+; AVX2-LABEL: combine_pmaddubsw_add_independent_inputs:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX2-NEXT: vpand %ymm5, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm1
+; AVX2-NEXT: vpand %ymm5, %ymm3, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_independent_inputs:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; AVX512-NEXT: vpand %ymm4, %ymm0, %ymm0
+; AVX512-NEXT: vpbroadcastb {{.*#+}} ymm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512-NEXT: vpand %ymm5, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpand %ymm4, %ymm2, %ymm1
+; AVX512-NEXT: vpand %ymm5, %ymm3, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm1, %ymm1
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: retq
+ %and0 = and <32 x i8> %a0, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and1 = and <32 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %and2 = and <32 x i8> %a2, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and3 = and <32 x i8> %a3, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %m0 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %and0, <32 x i8> %and1)
+ %m1 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %and2, <32 x i8> %and3)
+ %res = add <16 x i16> %m0, %m1
+ ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>)
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
new file mode 100644
index 000000000000000..54e21d41a8a49cb
--- /dev/null
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
@@ -0,0 +1,134 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX512
+
+; 255 * 63 + 255 * 63 fits in signed i16.
+; The unsigned input may have its high bit set; keep it as operand 0.
+define <32 x i16> @combine_pmaddubsw_add_u8_s6(<64 x i8> %a0, <64 x i8> %a1) {
+; AVX512-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm3
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm2
+; AVX512-NEXT: vpmaddubsw %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
+; AVX512-NEXT: vpsrlw $8, %zmm1, %zmm1
+; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: retq
+ %and = and <64 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %even1 = shufflevector <64 x i8> %and, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %odd0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %odd1 = shufflevector <64 x i8> %and, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %even = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %even0, <64 x i8> %even1)
+ %odd = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %odd0, <64 x i8> %odd1)
+ %res = add <32 x i16> %even, %odd
+ ret <32 x i16> %res
+}
+
+; The ADD operands can be commuted without changing PMADDUBSW operand roles.
+define <32 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<64 x i8> %a0, <64 x i8> %a1) {
+; AVX512-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm3
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm2
+; AVX512-NEXT: vpmaddubsw %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
+; AVX512-NEXT: vpsrlw $8, %zmm1, %zmm1
+; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: retq
+ %and = and <64 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %even1 = shufflevector <64 x i8> %and, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %odd0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %odd1 = shufflevector <64 x i8> %and, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %even = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %even0, <64 x i8> %even1)
+ %odd = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %odd0, <64 x i8> %odd1)
+ %res = add <32 x i16> %odd, %even
+ ret <32 x i16> %res
+}
+
+; Only the last output lane can overflow: 255 * 127 + 255 * 127 wraps to
+; -766, but a combined PMADDUBSW saturates to 32767. All other lanes are
+; bounded by 127 * 127 + 127 * 127.
+define <32 x i16> @combine_pmaddubsw_add_positive_overflow(<64 x i8> %a0, <64 x i8> %a1) {
+; AVX512-LABEL: combine_pmaddubsw_add_positive_overflow:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm3
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm2
+; AVX512-NEXT: vpmaddubsw %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
+; AVX512-NEXT: vpsrlw $8, %zmm1, %zmm1
+; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: retq
+ %and0 = and <64 x i8> %a0, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 -1, i8 -1>
+ %and1 = and <64 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %even0 = shufflevector <64 x i8> %and0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %even1 = shufflevector <64 x i8> %and1, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %odd0 = shufflevector <64 x i8> %and0, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %odd1 = shufflevector <64 x i8> %and1, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %even = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %even0, <64 x i8> %even1)
+ %odd = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %odd0, <64 x i8> %odd1)
+ %res = add <32 x i16> %even, %odd
+ ret <32 x i16> %res
+}
+
+; 255 * -128 + 255 * -128 wraps to 256, but a combined PMADDUBSW saturates to -32768.
+define <32 x i16> @combine_pmaddubsw_add_negative_overflow(<64 x i8> %a0, <64 x i8> %a1) {
+; AVX512-LABEL: combine_pmaddubsw_add_negative_overflow:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm3
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm2
+; AVX512-NEXT: vpmaddubsw %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
+; AVX512-NEXT: vpsrlw $8, %zmm1, %zmm1
+; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: retq
+ %or = or <64 x i8> %a1, <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>
+ %even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %even1 = shufflevector <64 x i8> %or, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %odd0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %odd1 = shufflevector <64 x i8> %or, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %even = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %even0, <64 x i8> %even1)
+ %odd = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %odd0, <64 x i8> %odd1)
+ %res = add <32 x i16> %even, %odd
+ ret <32 x i16> %res
+}
+
+; Independent inputs require new shuffles. Preserve the two PMADDUBSWs
+; and the ADD instead of introducing those shuffles.
+define <32 x i16> @combine_pmaddubsw_add_independent_inputs(<64 x i8> %a0, <64 x i8> %a1, <64 x i8> %a2, <64 x i8> %a3) {
+; AVX512-LABEL: combine_pmaddubsw_add_independent_inputs:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; AVX512-NEXT: vpandq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT: vpbroadcastb {{.*#+}} zmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512-NEXT: vpandq %zmm5, %zmm1, %zmm1
+; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpandq %zmm4, %zmm2, %zmm1
+; AVX512-NEXT: vpandq %zmm5, %zmm3, %zmm2
+; AVX512-NEXT: vpmaddubsw %zmm2, %zmm1, %zmm1
+; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: retq
+ %and0 = and <64 x i8> %a0, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and1 = and <64 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %and2 = and <64 x i8> %a2, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and3 = and <64 x i8> %a3, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %m0 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %and0, <64 x i8> %and1)
+ %m1 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %and2, <64 x i8> %and3)
+ %res = add <32 x i16> %m0, %m1
+ ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8>, <64 x i8>)
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
index 747f69ea61e54bb..d6849eefac8f9bc 100644
--- a/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
@@ -168,6 +168,332 @@ define i32 @combine_pmaddubsw_constant_sat() {
%3 = sext i16 %2 to i32
ret i32 %3
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; SSE41: {{.*}}
-; SSSE3: {{.*}}
+
+; 255 * 63 + 255 * 63 fits in signed i16.
+; The unsigned input may have its high bit set; keep it as operand 0.
+define <8 x i16> @combine_pmaddubsw_add_u8_s6(<16 x i8> %a0, <16 x i8> %a1) {
+; SSSE3-LABEL: combine_pmaddubsw_add_u8_s6:
+; SSSE3: # %bb.0:
+; SSSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pand %xmm2, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm2
+; SSSE3-NEXT: pmaddubsw %xmm2, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm0
+; SSSE3-NEXT: psrlw $8, %xmm1
+; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
+; SSSE3-NEXT: paddw %xmm3, %xmm0
+; SSSE3-NEXT: retq
+;
+; SSE41-LABEL: combine_pmaddubsw_add_u8_s6:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pand %xmm2, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm2
+; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm0
+; SSE41-NEXT: psrlw $8, %xmm1
+; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE41-NEXT: paddw %xmm3, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX1-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX1-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX2-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: retq
+ %and = and <16 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %odd0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %odd1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %even = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %even0, <16 x i8> %even1)
+ %odd = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %odd0, <16 x i8> %odd1)
+ %res = add <8 x i16> %even, %odd
+ ret <8 x i16> %res
+}
+
+; The ADD operands can be commuted without changing PMADDUBSW operand roles.
+define <8 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<16 x i8> %a0, <16 x i8> %a1) {
+; SSSE3-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; SSSE3: # %bb.0:
+; SSSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pand %xmm2, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm2
+; SSSE3-NEXT: pmaddubsw %xmm2, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm0
+; SSSE3-NEXT: psrlw $8, %xmm1
+; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
+; SSSE3-NEXT: paddw %xmm3, %xmm0
+; SSSE3-NEXT: retq
+;
+; SSE41-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pand %xmm2, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm2
+; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm0
+; SSE41-NEXT: psrlw $8, %xmm1
+; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE41-NEXT: paddw %xmm3, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX1-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX1-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX2-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: retq
+ %and = and <16 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %odd0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %odd1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %even = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %even0, <16 x i8> %even1)
+ %odd = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %odd0, <16 x i8> %odd1)
+ %res = add <8 x i16> %odd, %even
+ ret <8 x i16> %res
+}
+
+; 255 * 127 + 255 * 127 wraps to -766, but a combined PMADDUBSW saturates to 32767.
+define <8 x i16> @combine_pmaddubsw_add_positive_overflow(<16 x i8> %a0, <16 x i8> %a1) {
+; SSSE3-LABEL: combine_pmaddubsw_add_positive_overflow:
+; SSSE3: # %bb.0:
+; SSSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pand %xmm2, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm2
+; SSSE3-NEXT: pmaddubsw %xmm2, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm0
+; SSSE3-NEXT: psrlw $8, %xmm1
+; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
+; SSSE3-NEXT: paddw %xmm3, %xmm0
+; SSSE3-NEXT: retq
+;
+; SSE41-LABEL: combine_pmaddubsw_add_positive_overflow:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pand %xmm2, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm2
+; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm0
+; SSE41-NEXT: psrlw $8, %xmm1
+; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE41-NEXT: paddw %xmm3, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX1-LABEL: combine_pmaddubsw_add_positive_overflow:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX1-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: combine_pmaddubsw_add_positive_overflow:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX2-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: retq
+ %and = and <16 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %odd0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %odd1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %even = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %even0, <16 x i8> %even1)
+ %odd = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %odd0, <16 x i8> %odd1)
+ %res = add <8 x i16> %even, %odd
+ ret <8 x i16> %res
+}
+
+; 255 * -128 + 255 * -128 wraps to 256, but a combined PMADDUBSW saturates to -32768.
+define <8 x i16> @combine_pmaddubsw_add_negative_overflow(<16 x i8> %a0, <16 x i8> %a1) {
+; SSSE3-LABEL: combine_pmaddubsw_add_negative_overflow:
+; SSSE3: # %bb.0:
+; SSSE3-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pand %xmm2, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm2
+; SSSE3-NEXT: pmaddubsw %xmm2, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm0
+; SSSE3-NEXT: psrlw $8, %xmm1
+; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
+; SSSE3-NEXT: paddw %xmm3, %xmm0
+; SSSE3-NEXT: retq
+;
+; SSE41-LABEL: combine_pmaddubsw_add_negative_overflow:
+; SSE41: # %bb.0:
+; SSE41-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pand %xmm2, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm2
+; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm0
+; SSE41-NEXT: psrlw $8, %xmm1
+; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE41-NEXT: paddw %xmm3, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX1-LABEL: combine_pmaddubsw_add_negative_overflow:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX1-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: combine_pmaddubsw_add_negative_overflow:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX2-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: retq
+ %or = or <16 x i8> %a1, <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>
+ %even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %even1 = shufflevector <16 x i8> %or, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %odd0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %odd1 = shufflevector <16 x i8> %or, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %even = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %even0, <16 x i8> %even1)
+ %odd = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %odd0, <16 x i8> %odd1)
+ %res = add <8 x i16> %even, %odd
+ ret <8 x i16> %res
+}
+
+; Independent inputs require new shuffles. Preserve the two PMADDUBSWs
+; and the ADD instead of introducing those shuffles.
+define <8 x i16> @combine_pmaddubsw_add_independent_inputs(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2, <16 x i8> %a3) {
+; SSSE3-LABEL: combine_pmaddubsw_add_independent_inputs:
+; SSSE3: # %bb.0:
+; SSSE3-NEXT: movdqa {{.*#+}} xmm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; SSSE3-NEXT: pand %xmm4, %xmm0
+; SSSE3-NEXT: movdqa {{.*#+}} xmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSSE3-NEXT: pand %xmm5, %xmm1
+; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
+; SSSE3-NEXT: pand %xmm2, %xmm4
+; SSSE3-NEXT: pand %xmm5, %xmm3
+; SSSE3-NEXT: pmaddubsw %xmm3, %xmm4
+; SSSE3-NEXT: paddw %xmm4, %xmm0
+; SSSE3-NEXT: retq
+;
+; SSE41-LABEL: combine_pmaddubsw_add_independent_inputs:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pmovsxbw {{.*#+}} xmm4 = [127,127,127,127,127,127,127,127]
+; SSE41-NEXT: pand %xmm4, %xmm0
+; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE41-NEXT: pand %xmm5, %xmm1
+; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE41-NEXT: pand %xmm2, %xmm4
+; SSE41-NEXT: pand %xmm5, %xmm3
+; SSE41-NEXT: pmaddubsw %xmm3, %xmm4
+; SSE41-NEXT: paddw %xmm4, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX1-LABEL: combine_pmaddubsw_add_independent_inputs:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX1-NEXT: vpand %xmm5, %xmm1, %xmm1
+; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm4, %xmm2, %xmm1
+; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm2
+; AVX1-NEXT: vpmaddubsw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: combine_pmaddubsw_add_independent_inputs:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; AVX2-NEXT: vpand %xmm4, %xmm0, %xmm0
+; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX2-NEXT: vpand %xmm5, %xmm1, %xmm1
+; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpand %xmm4, %xmm2, %xmm1
+; AVX2-NEXT: vpand %xmm5, %xmm3, %xmm2
+; AVX2-NEXT: vpmaddubsw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: retq
+ %and0 = and <16 x i8> %a0, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and1 = and <16 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %and2 = and <16 x i8> %a2, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and3 = and <16 x i8> %a3, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %m0 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %and0, <16 x i8> %and1)
+ %m1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %and2, <16 x i8> %and3)
+ %res = add <8 x i16> %m0, %m1
+ ret <8 x i16> %res
+}
>From e7ec162b37bb3c2b69cdd3d07e543dc36175acfa Mon Sep 17 00:00:00 2001
From: Jihyeon Jeong <jh.jeong129 at gmail.com>
Date: Fri, 2 Oct 2026 04:13:40 +0000
Subject: [PATCH 2/2] [X86] Combine adds of PMADDUBSW with zero upper products
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 81 ++++++++++--
.../test/CodeGen/X86/combine-pmaddubsw-256.ll | 28 ----
.../test/CodeGen/X86/combine-pmaddubsw-512.ll | 14 --
llvm/test/CodeGen/X86/combine-pmaddubsw.ll | 124 +++---------------
4 files changed, 90 insertions(+), 157 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 5a7235d7249a3ff..1f5d641d8d2597e 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -59575,17 +59575,52 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N,
PMADDBuilder);
}
-// ADD(VPMADDWD(X,Y),VPMADDWD(Z,W)) -> VPMADDWD(SHUFFLE(X,Z), SHUFFLE(Y,W))
-// If upper element in each pair of both VPMADDWD are zero then we can merge
-// the operand elements and use the implicit add of VPMADDWD.
-// TODO: Add support for VPMADDUBSW (which isn't commutable).
-static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
- const SDLoc &DL, EVT VT) {
- if (N0.getOpcode() != N1.getOpcode() || N0.getOpcode() != X86ISD::VPMADDWD)
+// Return an existing input if interleaving the even elements of EvenOp and
+// OddOp reconstructs it without any additional shuffling.
+static SDValue getInterleavedPMADDInput(SelectionDAG &DAG, SDValue EvenOp,
+ SDValue OddOp) {
+ MVT VT = EvenOp.getSimpleValueType();
+ unsigned NumElts = VT.getVectorNumElements();
+ SDValue Src;
+ for (unsigned I = 0; I != 2; ++I) {
+ SDValue Op = peekThroughBitcasts(I == 0 ? EvenOp : OddOp);
+ SmallVector<SDValue, 2> Inputs;
+ SmallVector<int, 64> Mask, ScaledMask;
+ if (!getTargetShuffleInputs(Op, Inputs, Mask, DAG)) {
+ Inputs.assign(1, Op);
+ Mask.resize(NumElts);
+ std::iota(Mask.begin(), Mask.end(), 0);
+ }
+ if (!scaleShuffleMaskElts(NumElts, Mask, ScaledMask))
+ return SDValue();
+
+ for (unsigned J = 0; J != NumElts; J += 2) {
+ int M = ScaledMask[J];
+ if (M < 0 || unsigned(M) % NumElts != J + I)
+ return SDValue();
+ SDValue Input = peekThroughBitcasts(Inputs[M / NumElts]);
+ if (!Input.getValueType().isVector() ||
+ Input.getValueSizeInBits() != VT.getSizeInBits() ||
+ (Src && Src != Input))
+ return SDValue();
+ Src = Input;
+ }
+ }
+ return Src;
+}
+
+// ADD(PMADD(X,Y),PMADD(Z,W)) -> PMADD(SHUFFLE(X,Z), SHUFFLE(Y,W))
+// If the upper product in each pair of both PMADDs is zero, merge the operand
+// elements and use the implicit add of PMADD.
+static SDValue combineAddOfPMADD(SelectionDAG &DAG, SDValue N0, SDValue N1,
+ const SDLoc &DL, EVT VT) {
+ if (N0.getOpcode() != N1.getOpcode() ||
+ (N0.getOpcode() != X86ISD::VPMADDWD &&
+ N0.getOpcode() != X86ISD::VPMADDUBSW))
return SDValue();
// TODO: Add 256/512-bit support once VPMADDWD combines with shuffles.
- if (VT.getSizeInBits() > 128)
+ if (N0.getOpcode() == X86ISD::VPMADDWD && VT.getSizeInBits() > 128)
return SDValue();
unsigned NumElts = VT.getVectorNumElements();
@@ -59602,7 +59637,31 @@ static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
if (!Op0HiZero || !Op1HiZero)
return SDValue();
- // Create a shuffle mask packing the lower elements from each VPMADDWD.
+ // Each unsigned i8 * signed i8 product fits in signed i16. Ensure their sum
+ // does too, so that the saturation in VPMADDUBSW does not change the result.
+ if (N0.getOpcode() == X86ISD::VPMADDUBSW &&
+ !DAG.willNotOverflowAdd(/*IsSigned=*/true, N0, N1))
+ return SDValue();
+
+ // Only combine PMADDUBSW when both interleaves reconstruct existing inputs;
+ // introducing new shuffles can make this combine unprofitable.
+ if (N0.getOpcode() == X86ISD::VPMADDUBSW) {
+ // Try both ADD operand orders, keeping the unsigned/signed operand order
+ // of each PMADDUBSW and matching both inputs in the same direction.
+ for (unsigned I = 0; I != 2; ++I) {
+ SDValue LHS =
+ getInterleavedPMADDInput(DAG, N0.getOperand(0), N1.getOperand(0));
+ SDValue RHS =
+ getInterleavedPMADDInput(DAG, N0.getOperand(1), N1.getOperand(1));
+ if (LHS && RHS)
+ return DAG.getNode(N0.getOpcode(), DL, VT, DAG.getBitcast(OpVT, LHS),
+ DAG.getBitcast(OpVT, RHS));
+ std::swap(N0, N1);
+ }
+ return SDValue();
+ }
+
+ // Create a shuffle mask packing the lower elements from each PMADD.
SmallVector<int> Mask;
for (int i = 0; i != (int)NumElts; ++i) {
Mask.push_back(2 * i);
@@ -59613,7 +59672,7 @@ static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
DAG.getVectorShuffle(OpVT, DL, N0.getOperand(0), N1.getOperand(0), Mask);
SDValue RHS =
DAG.getVectorShuffle(OpVT, DL, N0.getOperand(1), N1.getOperand(1), Mask);
- return DAG.getNode(X86ISD::VPMADDWD, DL, VT, LHS, RHS);
+ return DAG.getNode(N0.getOpcode(), DL, VT, LHS, RHS);
}
/// CMOV of constants requires materializing constant operands in registers.
@@ -59742,7 +59801,7 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
return MAdd;
if (SDValue MAdd = matchPMADDWD_2(DAG, N, DL, VT, Subtarget))
return MAdd;
- if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT))
+ if (SDValue MAdd = combineAddOfPMADD(DAG, Op0, Op1, DL, VT))
return MAdd;
// Try to synthesize horizontal adds from adds of shuffles.
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
index 082b3b60bb91afc..37a4f475ebaba9a 100644
--- a/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
@@ -8,27 +8,13 @@ define <16 x i16> @combine_pmaddubsw_add_u8_s6(<32 x i8> %a0, <32 x i8> %a1) {
; AVX2-LABEL: combine_pmaddubsw_add_u8_s6:
; AVX2: # %bb.0:
; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
-; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
-; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
-; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpaddw %ymm0, %ymm2, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: combine_pmaddubsw_add_u8_s6:
; AVX512: # %bb.0:
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
-; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
-; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
-; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
-; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
-; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpaddw %ymm0, %ymm2, %ymm0
; AVX512-NEXT: retq
%and = and <32 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
%even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
@@ -46,27 +32,13 @@ define <16 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<32 x i8> %a0, <32 x i8>
; AVX2-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
; AVX2: # %bb.0:
; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
-; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
-; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
-; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpaddw %ymm2, %ymm0, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
; AVX512: # %bb.0:
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
-; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
-; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
-; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
-; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
-; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpaddw %ymm2, %ymm0, %ymm0
; AVX512-NEXT: retq
%and = and <32 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
%even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
index 54e21d41a8a49cb..4daf64f3c910c3a 100644
--- a/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
@@ -7,14 +7,7 @@ define <32 x i16> @combine_pmaddubsw_add_u8_s6(<64 x i8> %a0, <64 x i8> %a1) {
; AVX512-LABEL: combine_pmaddubsw_add_u8_s6:
; AVX512: # %bb.0:
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
-; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm3
-; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm2
-; AVX512-NEXT: vpmaddubsw %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512-NEXT: vpsrlw $8, %zmm1, %zmm1
; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpaddw %zmm0, %zmm2, %zmm0
; AVX512-NEXT: retq
%and = and <64 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
%even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
@@ -32,14 +25,7 @@ define <32 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<64 x i8> %a0, <64 x i8>
; AVX512-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
; AVX512: # %bb.0:
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
-; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm3
-; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm2
-; AVX512-NEXT: vpmaddubsw %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512-NEXT: vpsrlw $8, %zmm1, %zmm1
; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpaddw %zmm2, %zmm0, %zmm0
; AVX512-NEXT: retq
%and = and <64 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
%even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
index d6849eefac8f9bc..61790d2d08a3b18 100644
--- a/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
@@ -172,59 +172,17 @@ define i32 @combine_pmaddubsw_constant_sat() {
; 255 * 63 + 255 * 63 fits in signed i16.
; The unsigned input may have its high bit set; keep it as operand 0.
define <8 x i16> @combine_pmaddubsw_add_u8_s6(<16 x i8> %a0, <16 x i8> %a1) {
-; SSSE3-LABEL: combine_pmaddubsw_add_u8_s6:
-; SSSE3: # %bb.0:
-; SSSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSSE3-NEXT: movdqa %xmm0, %xmm3
-; SSSE3-NEXT: pand %xmm2, %xmm3
-; SSSE3-NEXT: pand %xmm1, %xmm2
-; SSSE3-NEXT: pmaddubsw %xmm2, %xmm3
-; SSSE3-NEXT: psrlw $8, %xmm0
-; SSSE3-NEXT: psrlw $8, %xmm1
-; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
-; SSSE3-NEXT: paddw %xmm3, %xmm0
-; SSSE3-NEXT: retq
-;
-; SSE41-LABEL: combine_pmaddubsw_add_u8_s6:
-; SSE41: # %bb.0:
-; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT: movdqa %xmm0, %xmm3
-; SSE41-NEXT: pand %xmm2, %xmm3
-; SSE41-NEXT: pand %xmm1, %xmm2
-; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
-; SSE41-NEXT: psrlw $8, %xmm0
-; SSE41-NEXT: psrlw $8, %xmm1
-; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
-; SSE41-NEXT: paddw %xmm3, %xmm0
-; SSE41-NEXT: retq
-;
-; AVX1-LABEL: combine_pmaddubsw_add_u8_s6:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
-; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm2
-; AVX1-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpaddw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: retq
+; SSE-LABEL: combine_pmaddubsw_add_u8_s6:
+; SSE: # %bb.0:
+; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE-NEXT: retq
;
-; AVX2-LABEL: combine_pmaddubsw_add_u8_s6:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
-; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm2
-; AVX2-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
-; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpaddw %xmm0, %xmm2, %xmm0
-; AVX2-NEXT: retq
+; AVX-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX: # %bb.0:
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
%and = and <16 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
%even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
%even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
@@ -238,59 +196,17 @@ define <8 x i16> @combine_pmaddubsw_add_u8_s6(<16 x i8> %a0, <16 x i8> %a1) {
; The ADD operands can be commuted without changing PMADDUBSW operand roles.
define <8 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<16 x i8> %a0, <16 x i8> %a1) {
-; SSSE3-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
-; SSSE3: # %bb.0:
-; SSSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSSE3-NEXT: movdqa %xmm0, %xmm3
-; SSSE3-NEXT: pand %xmm2, %xmm3
-; SSSE3-NEXT: pand %xmm1, %xmm2
-; SSSE3-NEXT: pmaddubsw %xmm2, %xmm3
-; SSSE3-NEXT: psrlw $8, %xmm0
-; SSSE3-NEXT: psrlw $8, %xmm1
-; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
-; SSSE3-NEXT: paddw %xmm3, %xmm0
-; SSSE3-NEXT: retq
-;
-; SSE41-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
-; SSE41: # %bb.0:
-; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT: movdqa %xmm0, %xmm3
-; SSE41-NEXT: pand %xmm2, %xmm3
-; SSE41-NEXT: pand %xmm1, %xmm2
-; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
-; SSE41-NEXT: psrlw $8, %xmm0
-; SSE41-NEXT: psrlw $8, %xmm1
-; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
-; SSE41-NEXT: paddw %xmm3, %xmm0
-; SSE41-NEXT: retq
-;
-; AVX1-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
-; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm2
-; AVX1-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpaddw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: retq
+; SSE-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; SSE: # %bb.0:
+; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE-NEXT: retq
;
-; AVX2-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
-; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm2
-; AVX2-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
-; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpaddw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: retq
+; AVX-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX: # %bb.0:
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
%and = and <16 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
%even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
%even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
More information about the llvm-commits
mailing list