[llvm] [X86] combineAddOfPMADDWD - add PMADDUBSW support (PR #228359)
Jeong Jihyeon via llvm-commits
llvm-commits at lists.llvm.org
Fri Oct 2 03:39:25 PDT 2026
https://github.com/JihyeonJeong129 updated https://github.com/llvm/llvm-project/pull/228359
>From 06498797aebf2c91e72f1a49d01cb0a99fe781ea Mon Sep 17 00:00:00 2001
From: Jihyeon Jeong <jh.jeong129 at gmail.com>
Date: Fri, 2 Oct 2026 04:11:41 +0000
Subject: [PATCH 1/3] [X86] Add tests for add-of-PMADDUBSW combines
---
.../test/CodeGen/X86/combine-pmaddubsw-256.ll | 201 +++++++++++
.../test/CodeGen/X86/combine-pmaddubsw-512.ll | 134 +++++++
llvm/test/CodeGen/X86/combine-pmaddubsw.ll | 332 +++++++++++++++++-
3 files changed, 664 insertions(+), 3 deletions(-)
create mode 100644 llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
create mode 100644 llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
new file mode 100644
index 0000000000000..082b3b60bb91a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
@@ -0,0 +1,201 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw,+avx512vl | FileCheck %s --check-prefix=AVX512
+
+; 255 * 63 + 255 * 63 fits in signed i16.
+; The unsigned input may have its high bit set; keep it as operand 0.
+define <16 x i16> @combine_pmaddubsw_add_u8_s6(<32 x i8> %a0, <32 x i8> %a1) {
+; AVX2-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX512-NEXT: retq
+ %and = and <32 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %even1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %odd0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %odd1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %even = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %even0, <32 x i8> %even1)
+ %odd = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %odd0, <32 x i8> %odd1)
+ %res = add <16 x i16> %even, %odd
+ ret <16 x i16> %res
+}
+
+; The ADD operands can be commuted without changing PMADDUBSW operand roles.
+define <16 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<32 x i8> %a0, <32 x i8> %a1) {
+; AVX2-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm2, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddw %ymm2, %ymm0, %ymm0
+; AVX512-NEXT: retq
+ %and = and <32 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %even1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %odd0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %odd1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %even = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %even0, <32 x i8> %even1)
+ %odd = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %odd0, <32 x i8> %odd1)
+ %res = add <16 x i16> %odd, %even
+ ret <16 x i16> %res
+}
+
+; Only the last output lane can overflow: 255 * 127 + 255 * 127 wraps to
+; -766, but a combined PMADDUBSW saturates to 32767. All other lanes are
+; bounded by 127 * 127 + 127 * 127.
+define <16 x i16> @combine_pmaddubsw_add_positive_overflow(<32 x i8> %a0, <32 x i8> %a1) {
+; AVX2-LABEL: combine_pmaddubsw_add_positive_overflow:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_positive_overflow:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm0, %ymm0
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX512-NEXT: retq
+ %and0 = and <32 x i8> %a0, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 -1, i8 -1>
+ %and1 = and <32 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %even0 = shufflevector <32 x i8> %and0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %even1 = shufflevector <32 x i8> %and1, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %odd0 = shufflevector <32 x i8> %and0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %odd1 = shufflevector <32 x i8> %and1, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %even = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %even0, <32 x i8> %even1)
+ %odd = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %odd0, <32 x i8> %odd1)
+ %res = add <16 x i16> %even, %odd
+ ret <16 x i16> %res
+}
+
+; 255 * -128 + 255 * -128 wraps to 256, but a combined PMADDUBSW saturates to -32768.
+define <16 x i16> @combine_pmaddubsw_add_negative_overflow(<32 x i8> %a0, <32 x i8> %a1) {
+; AVX2-LABEL: combine_pmaddubsw_add_negative_overflow:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_negative_overflow:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
+; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
+; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
+; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpaddw %ymm0, %ymm2, %ymm0
+; AVX512-NEXT: retq
+ %or = or <32 x i8> %a1, <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>
+ %even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %even1 = shufflevector <32 x i8> %or, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
+ %odd0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %odd1 = shufflevector <32 x i8> %or, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
+ %even = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %even0, <32 x i8> %even1)
+ %odd = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %odd0, <32 x i8> %odd1)
+ %res = add <16 x i16> %even, %odd
+ ret <16 x i16> %res
+}
+
+; Independent inputs require new shuffles. Preserve the two PMADDUBSWs
+; and the ADD instead of introducing those shuffles.
+define <16 x i16> @combine_pmaddubsw_add_independent_inputs(<32 x i8> %a0, <32 x i8> %a1, <32 x i8> %a2, <32 x i8> %a3) {
+; AVX2-LABEL: combine_pmaddubsw_add_independent_inputs:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; AVX2-NEXT: vpand %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpbroadcastb {{.*#+}} ymm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX2-NEXT: vpand %ymm5, %ymm1, %ymm1
+; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: vpand %ymm4, %ymm2, %ymm1
+; AVX2-NEXT: vpand %ymm5, %ymm3, %ymm2
+; AVX2-NEXT: vpmaddubsw %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpaddw %ymm1, %ymm0, %ymm0
+; AVX2-NEXT: retq
+;
+; AVX512-LABEL: combine_pmaddubsw_add_independent_inputs:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; AVX512-NEXT: vpand %ymm4, %ymm0, %ymm0
+; AVX512-NEXT: vpbroadcastb {{.*#+}} ymm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512-NEXT: vpand %ymm5, %ymm1, %ymm1
+; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: vpand %ymm4, %ymm2, %ymm1
+; AVX512-NEXT: vpand %ymm5, %ymm3, %ymm2
+; AVX512-NEXT: vpmaddubsw %ymm2, %ymm1, %ymm1
+; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
+; AVX512-NEXT: retq
+ %and0 = and <32 x i8> %a0, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and1 = and <32 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %and2 = and <32 x i8> %a2, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and3 = and <32 x i8> %a3, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %m0 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %and0, <32 x i8> %and1)
+ %m1 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %and2, <32 x i8> %and3)
+ %res = add <16 x i16> %m0, %m1
+ ret <16 x i16> %res
+}
+
+declare <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8>, <32 x i8>)
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
new file mode 100644
index 0000000000000..54e21d41a8a49
--- /dev/null
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
@@ -0,0 +1,134 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefix=AVX512
+
+; 255 * 63 + 255 * 63 fits in signed i16.
+; The unsigned input may have its high bit set; keep it as operand 0.
+define <32 x i16> @combine_pmaddubsw_add_u8_s6(<64 x i8> %a0, <64 x i8> %a1) {
+; AVX512-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm3
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm2
+; AVX512-NEXT: vpmaddubsw %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
+; AVX512-NEXT: vpsrlw $8, %zmm1, %zmm1
+; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: retq
+ %and = and <64 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %even1 = shufflevector <64 x i8> %and, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %odd0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %odd1 = shufflevector <64 x i8> %and, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %even = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %even0, <64 x i8> %even1)
+ %odd = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %odd0, <64 x i8> %odd1)
+ %res = add <32 x i16> %even, %odd
+ ret <32 x i16> %res
+}
+
+; The ADD operands can be commuted without changing PMADDUBSW operand roles.
+define <32 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<64 x i8> %a0, <64 x i8> %a1) {
+; AVX512-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm3
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm2
+; AVX512-NEXT: vpmaddubsw %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
+; AVX512-NEXT: vpsrlw $8, %zmm1, %zmm1
+; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %zmm2, %zmm0, %zmm0
+; AVX512-NEXT: retq
+ %and = and <64 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %even1 = shufflevector <64 x i8> %and, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %odd0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %odd1 = shufflevector <64 x i8> %and, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %even = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %even0, <64 x i8> %even1)
+ %odd = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %odd0, <64 x i8> %odd1)
+ %res = add <32 x i16> %odd, %even
+ ret <32 x i16> %res
+}
+
+; Only the last output lane can overflow: 255 * 127 + 255 * 127 wraps to
+; -766, but a combined PMADDUBSW saturates to 32767. All other lanes are
+; bounded by 127 * 127 + 127 * 127.
+define <32 x i16> @combine_pmaddubsw_add_positive_overflow(<64 x i8> %a0, <64 x i8> %a1) {
+; AVX512-LABEL: combine_pmaddubsw_add_positive_overflow:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpandq {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %zmm0, %zmm0
+; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm3
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm2
+; AVX512-NEXT: vpmaddubsw %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
+; AVX512-NEXT: vpsrlw $8, %zmm1, %zmm1
+; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: retq
+ %and0 = and <64 x i8> %a0, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 -1, i8 -1>
+ %and1 = and <64 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %even0 = shufflevector <64 x i8> %and0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %even1 = shufflevector <64 x i8> %and1, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %odd0 = shufflevector <64 x i8> %and0, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %odd1 = shufflevector <64 x i8> %and1, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %even = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %even0, <64 x i8> %even1)
+ %odd = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %odd0, <64 x i8> %odd1)
+ %res = add <32 x i16> %even, %odd
+ ret <32 x i16> %res
+}
+
+; 255 * -128 + 255 * -128 wraps to 256, but a combined PMADDUBSW saturates to -32768.
+define <32 x i16> @combine_pmaddubsw_add_negative_overflow(<64 x i8> %a0, <64 x i8> %a1) {
+; AVX512-LABEL: combine_pmaddubsw_add_negative_overflow:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpord {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
+; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm3
+; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm2
+; AVX512-NEXT: vpmaddubsw %zmm2, %zmm3, %zmm2
+; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
+; AVX512-NEXT: vpsrlw $8, %zmm1, %zmm1
+; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpaddw %zmm0, %zmm2, %zmm0
+; AVX512-NEXT: retq
+ %or = or <64 x i8> %a1, <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>
+ %even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %even1 = shufflevector <64 x i8> %or, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
+ %odd0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %odd1 = shufflevector <64 x i8> %or, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
+ %even = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %even0, <64 x i8> %even1)
+ %odd = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %odd0, <64 x i8> %odd1)
+ %res = add <32 x i16> %even, %odd
+ ret <32 x i16> %res
+}
+
+; Independent inputs require new shuffles. Preserve the two PMADDUBSWs
+; and the ADD instead of introducing those shuffles.
+define <32 x i16> @combine_pmaddubsw_add_independent_inputs(<64 x i8> %a0, <64 x i8> %a1, <64 x i8> %a2, <64 x i8> %a3) {
+; AVX512-LABEL: combine_pmaddubsw_add_independent_inputs:
+; AVX512: # %bb.0:
+; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; AVX512-NEXT: vpandq %zmm4, %zmm0, %zmm0
+; AVX512-NEXT: vpbroadcastb {{.*#+}} zmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX512-NEXT: vpandq %zmm5, %zmm1, %zmm1
+; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: vpandq %zmm4, %zmm2, %zmm1
+; AVX512-NEXT: vpandq %zmm5, %zmm3, %zmm2
+; AVX512-NEXT: vpmaddubsw %zmm2, %zmm1, %zmm1
+; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
+; AVX512-NEXT: retq
+ %and0 = and <64 x i8> %a0, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and1 = and <64 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %and2 = and <64 x i8> %a2, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and3 = and <64 x i8> %a3, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %m0 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %and0, <64 x i8> %and1)
+ %m1 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %and2, <64 x i8> %and3)
+ %res = add <32 x i16> %m0, %m1
+ ret <32 x i16> %res
+}
+
+declare <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8>, <64 x i8>)
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
index 747f69ea61e54..d6849eefac8f9 100644
--- a/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
@@ -168,6 +168,332 @@ define i32 @combine_pmaddubsw_constant_sat() {
%3 = sext i16 %2 to i32
ret i32 %3
}
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; SSE41: {{.*}}
-; SSSE3: {{.*}}
+
+; 255 * 63 + 255 * 63 fits in signed i16.
+; The unsigned input may have its high bit set; keep it as operand 0.
+define <8 x i16> @combine_pmaddubsw_add_u8_s6(<16 x i8> %a0, <16 x i8> %a1) {
+; SSSE3-LABEL: combine_pmaddubsw_add_u8_s6:
+; SSSE3: # %bb.0:
+; SSSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pand %xmm2, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm2
+; SSSE3-NEXT: pmaddubsw %xmm2, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm0
+; SSSE3-NEXT: psrlw $8, %xmm1
+; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
+; SSSE3-NEXT: paddw %xmm3, %xmm0
+; SSSE3-NEXT: retq
+;
+; SSE41-LABEL: combine_pmaddubsw_add_u8_s6:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pand %xmm2, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm2
+; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm0
+; SSE41-NEXT: psrlw $8, %xmm1
+; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE41-NEXT: paddw %xmm3, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX1-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX1-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX2-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: retq
+ %and = and <16 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %odd0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %odd1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %even = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %even0, <16 x i8> %even1)
+ %odd = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %odd0, <16 x i8> %odd1)
+ %res = add <8 x i16> %even, %odd
+ ret <8 x i16> %res
+}
+
+; The ADD operands can be commuted without changing PMADDUBSW operand roles.
+define <8 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<16 x i8> %a0, <16 x i8> %a1) {
+; SSSE3-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; SSSE3: # %bb.0:
+; SSSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pand %xmm2, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm2
+; SSSE3-NEXT: pmaddubsw %xmm2, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm0
+; SSSE3-NEXT: psrlw $8, %xmm1
+; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
+; SSSE3-NEXT: paddw %xmm3, %xmm0
+; SSSE3-NEXT: retq
+;
+; SSE41-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pand %xmm2, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm2
+; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm0
+; SSE41-NEXT: psrlw $8, %xmm1
+; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE41-NEXT: paddw %xmm3, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX1-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX1-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX2-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm2, %xmm0, %xmm0
+; AVX2-NEXT: retq
+ %and = and <16 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %odd0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %odd1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %even = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %even0, <16 x i8> %even1)
+ %odd = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %odd0, <16 x i8> %odd1)
+ %res = add <8 x i16> %odd, %even
+ ret <8 x i16> %res
+}
+
+; 255 * 127 + 255 * 127 wraps to -766, but a combined PMADDUBSW saturates to 32767.
+define <8 x i16> @combine_pmaddubsw_add_positive_overflow(<16 x i8> %a0, <16 x i8> %a1) {
+; SSSE3-LABEL: combine_pmaddubsw_add_positive_overflow:
+; SSSE3: # %bb.0:
+; SSSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pand %xmm2, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm2
+; SSSE3-NEXT: pmaddubsw %xmm2, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm0
+; SSSE3-NEXT: psrlw $8, %xmm1
+; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
+; SSSE3-NEXT: paddw %xmm3, %xmm0
+; SSSE3-NEXT: retq
+;
+; SSE41-LABEL: combine_pmaddubsw_add_positive_overflow:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pand %xmm2, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm2
+; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm0
+; SSE41-NEXT: psrlw $8, %xmm1
+; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE41-NEXT: paddw %xmm3, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX1-LABEL: combine_pmaddubsw_add_positive_overflow:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX1-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: combine_pmaddubsw_add_positive_overflow:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX2-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: retq
+ %and = and <16 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %odd0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %odd1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %even = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %even0, <16 x i8> %even1)
+ %odd = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %odd0, <16 x i8> %odd1)
+ %res = add <8 x i16> %even, %odd
+ ret <8 x i16> %res
+}
+
+; 255 * -128 + 255 * -128 wraps to 256, but a combined PMADDUBSW saturates to -32768.
+define <8 x i16> @combine_pmaddubsw_add_negative_overflow(<16 x i8> %a0, <16 x i8> %a1) {
+; SSSE3-LABEL: combine_pmaddubsw_add_negative_overflow:
+; SSSE3: # %bb.0:
+; SSSE3-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; SSSE3-NEXT: movdqa %xmm0, %xmm3
+; SSSE3-NEXT: pand %xmm2, %xmm3
+; SSSE3-NEXT: pand %xmm1, %xmm2
+; SSSE3-NEXT: pmaddubsw %xmm2, %xmm3
+; SSSE3-NEXT: psrlw $8, %xmm0
+; SSSE3-NEXT: psrlw $8, %xmm1
+; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
+; SSSE3-NEXT: paddw %xmm3, %xmm0
+; SSSE3-NEXT: retq
+;
+; SSE41-LABEL: combine_pmaddubsw_add_negative_overflow:
+; SSE41: # %bb.0:
+; SSE41-NEXT: por {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
+; SSE41-NEXT: movdqa %xmm0, %xmm3
+; SSE41-NEXT: pand %xmm2, %xmm3
+; SSE41-NEXT: pand %xmm1, %xmm2
+; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
+; SSE41-NEXT: psrlw $8, %xmm0
+; SSE41-NEXT: psrlw $8, %xmm1
+; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE41-NEXT: paddw %xmm3, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX1-LABEL: combine_pmaddubsw_add_negative_overflow:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX1-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: combine_pmaddubsw_add_negative_overflow:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpor {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
+; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
+; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm2
+; AVX2-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
+; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
+; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
+; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpaddw %xmm0, %xmm2, %xmm0
+; AVX2-NEXT: retq
+ %or = or <16 x i8> %a1, <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>
+ %even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %even1 = shufflevector <16 x i8> %or, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
+ %odd0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %odd1 = shufflevector <16 x i8> %or, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
+ %even = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %even0, <16 x i8> %even1)
+ %odd = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %odd0, <16 x i8> %odd1)
+ %res = add <8 x i16> %even, %odd
+ ret <8 x i16> %res
+}
+
+; Independent inputs require new shuffles. Preserve the two PMADDUBSWs
+; and the ADD instead of introducing those shuffles.
+define <8 x i16> @combine_pmaddubsw_add_independent_inputs(<16 x i8> %a0, <16 x i8> %a1, <16 x i8> %a2, <16 x i8> %a3) {
+; SSSE3-LABEL: combine_pmaddubsw_add_independent_inputs:
+; SSSE3: # %bb.0:
+; SSSE3-NEXT: movdqa {{.*#+}} xmm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; SSSE3-NEXT: pand %xmm4, %xmm0
+; SSSE3-NEXT: movdqa {{.*#+}} xmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSSE3-NEXT: pand %xmm5, %xmm1
+; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
+; SSSE3-NEXT: pand %xmm2, %xmm4
+; SSSE3-NEXT: pand %xmm5, %xmm3
+; SSSE3-NEXT: pmaddubsw %xmm3, %xmm4
+; SSSE3-NEXT: paddw %xmm4, %xmm0
+; SSSE3-NEXT: retq
+;
+; SSE41-LABEL: combine_pmaddubsw_add_independent_inputs:
+; SSE41: # %bb.0:
+; SSE41-NEXT: pmovsxbw {{.*#+}} xmm4 = [127,127,127,127,127,127,127,127]
+; SSE41-NEXT: pand %xmm4, %xmm0
+; SSE41-NEXT: movdqa {{.*#+}} xmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; SSE41-NEXT: pand %xmm5, %xmm1
+; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE41-NEXT: pand %xmm2, %xmm4
+; SSE41-NEXT: pand %xmm5, %xmm3
+; SSE41-NEXT: pmaddubsw %xmm3, %xmm4
+; SSE41-NEXT: paddw %xmm4, %xmm0
+; SSE41-NEXT: retq
+;
+; AVX1-LABEL: combine_pmaddubsw_add_independent_inputs:
+; AVX1: # %bb.0:
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; AVX1-NEXT: vpand %xmm4, %xmm0, %xmm0
+; AVX1-NEXT: vbroadcastss {{.*#+}} xmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX1-NEXT: vpand %xmm5, %xmm1, %xmm1
+; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: vpand %xmm4, %xmm2, %xmm1
+; AVX1-NEXT: vpand %xmm5, %xmm3, %xmm2
+; AVX1-NEXT: vpmaddubsw %xmm2, %xmm1, %xmm1
+; AVX1-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX1-NEXT: retq
+;
+; AVX2-LABEL: combine_pmaddubsw_add_independent_inputs:
+; AVX2: # %bb.0:
+; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm4 = [127,0,127,0,127,0,127,0,127,0,127,0,127,0,127,0]
+; AVX2-NEXT: vpand %xmm4, %xmm0, %xmm0
+; AVX2-NEXT: vpbroadcastb {{.*#+}} xmm5 = [127,127,127,127,127,127,127,127,127,127,127,127,127,127,127,127]
+; AVX2-NEXT: vpand %xmm5, %xmm1, %xmm1
+; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpand %xmm4, %xmm2, %xmm1
+; AVX2-NEXT: vpand %xmm5, %xmm3, %xmm2
+; AVX2-NEXT: vpmaddubsw %xmm2, %xmm1, %xmm1
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: retq
+ %and0 = and <16 x i8> %a0, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and1 = and <16 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %and2 = and <16 x i8> %a2, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
+ %and3 = and <16 x i8> %a3, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %m0 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %and0, <16 x i8> %and1)
+ %m1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %and2, <16 x i8> %and3)
+ %res = add <8 x i16> %m0, %m1
+ ret <8 x i16> %res
+}
>From e7ec162b37bb3c2b69cdd3d07e543dc36175acfa Mon Sep 17 00:00:00 2001
From: Jihyeon Jeong <jh.jeong129 at gmail.com>
Date: Fri, 2 Oct 2026 04:13:40 +0000
Subject: [PATCH 2/3] [X86] Combine adds of PMADDUBSW with zero upper products
---
llvm/lib/Target/X86/X86ISelLowering.cpp | 81 ++++++++++--
.../test/CodeGen/X86/combine-pmaddubsw-256.ll | 28 ----
.../test/CodeGen/X86/combine-pmaddubsw-512.ll | 14 --
llvm/test/CodeGen/X86/combine-pmaddubsw.ll | 124 +++---------------
4 files changed, 90 insertions(+), 157 deletions(-)
diff --git a/llvm/lib/Target/X86/X86ISelLowering.cpp b/llvm/lib/Target/X86/X86ISelLowering.cpp
index 5a7235d7249a3..1f5d641d8d259 100644
--- a/llvm/lib/Target/X86/X86ISelLowering.cpp
+++ b/llvm/lib/Target/X86/X86ISelLowering.cpp
@@ -59575,17 +59575,52 @@ static SDValue matchPMADDWD_2(SelectionDAG &DAG, SDNode *N,
PMADDBuilder);
}
-// ADD(VPMADDWD(X,Y),VPMADDWD(Z,W)) -> VPMADDWD(SHUFFLE(X,Z), SHUFFLE(Y,W))
-// If upper element in each pair of both VPMADDWD are zero then we can merge
-// the operand elements and use the implicit add of VPMADDWD.
-// TODO: Add support for VPMADDUBSW (which isn't commutable).
-static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
- const SDLoc &DL, EVT VT) {
- if (N0.getOpcode() != N1.getOpcode() || N0.getOpcode() != X86ISD::VPMADDWD)
+// Return an existing input if interleaving the even elements of EvenOp and
+// OddOp reconstructs it without any additional shuffling.
+static SDValue getInterleavedPMADDInput(SelectionDAG &DAG, SDValue EvenOp,
+ SDValue OddOp) {
+ MVT VT = EvenOp.getSimpleValueType();
+ unsigned NumElts = VT.getVectorNumElements();
+ SDValue Src;
+ for (unsigned I = 0; I != 2; ++I) {
+ SDValue Op = peekThroughBitcasts(I == 0 ? EvenOp : OddOp);
+ SmallVector<SDValue, 2> Inputs;
+ SmallVector<int, 64> Mask, ScaledMask;
+ if (!getTargetShuffleInputs(Op, Inputs, Mask, DAG)) {
+ Inputs.assign(1, Op);
+ Mask.resize(NumElts);
+ std::iota(Mask.begin(), Mask.end(), 0);
+ }
+ if (!scaleShuffleMaskElts(NumElts, Mask, ScaledMask))
+ return SDValue();
+
+ for (unsigned J = 0; J != NumElts; J += 2) {
+ int M = ScaledMask[J];
+ if (M < 0 || unsigned(M) % NumElts != J + I)
+ return SDValue();
+ SDValue Input = peekThroughBitcasts(Inputs[M / NumElts]);
+ if (!Input.getValueType().isVector() ||
+ Input.getValueSizeInBits() != VT.getSizeInBits() ||
+ (Src && Src != Input))
+ return SDValue();
+ Src = Input;
+ }
+ }
+ return Src;
+}
+
+// ADD(PMADD(X,Y),PMADD(Z,W)) -> PMADD(SHUFFLE(X,Z), SHUFFLE(Y,W))
+// If the upper product in each pair of both PMADDs is zero, merge the operand
+// elements and use the implicit add of PMADD.
+static SDValue combineAddOfPMADD(SelectionDAG &DAG, SDValue N0, SDValue N1,
+ const SDLoc &DL, EVT VT) {
+ if (N0.getOpcode() != N1.getOpcode() ||
+ (N0.getOpcode() != X86ISD::VPMADDWD &&
+ N0.getOpcode() != X86ISD::VPMADDUBSW))
return SDValue();
// TODO: Add 256/512-bit support once VPMADDWD combines with shuffles.
- if (VT.getSizeInBits() > 128)
+ if (N0.getOpcode() == X86ISD::VPMADDWD && VT.getSizeInBits() > 128)
return SDValue();
unsigned NumElts = VT.getVectorNumElements();
@@ -59602,7 +59637,31 @@ static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
if (!Op0HiZero || !Op1HiZero)
return SDValue();
- // Create a shuffle mask packing the lower elements from each VPMADDWD.
+ // Each unsigned i8 * signed i8 product fits in signed i16. Ensure their sum
+ // does too, so that the saturation in VPMADDUBSW does not change the result.
+ if (N0.getOpcode() == X86ISD::VPMADDUBSW &&
+ !DAG.willNotOverflowAdd(/*IsSigned=*/true, N0, N1))
+ return SDValue();
+
+ // Only combine PMADDUBSW when both interleaves reconstruct existing inputs;
+ // introducing new shuffles can make this combine unprofitable.
+ if (N0.getOpcode() == X86ISD::VPMADDUBSW) {
+ // Try both ADD operand orders, keeping the unsigned/signed operand order
+ // of each PMADDUBSW and matching both inputs in the same direction.
+ for (unsigned I = 0; I != 2; ++I) {
+ SDValue LHS =
+ getInterleavedPMADDInput(DAG, N0.getOperand(0), N1.getOperand(0));
+ SDValue RHS =
+ getInterleavedPMADDInput(DAG, N0.getOperand(1), N1.getOperand(1));
+ if (LHS && RHS)
+ return DAG.getNode(N0.getOpcode(), DL, VT, DAG.getBitcast(OpVT, LHS),
+ DAG.getBitcast(OpVT, RHS));
+ std::swap(N0, N1);
+ }
+ return SDValue();
+ }
+
+ // Create a shuffle mask packing the lower elements from each PMADD.
SmallVector<int> Mask;
for (int i = 0; i != (int)NumElts; ++i) {
Mask.push_back(2 * i);
@@ -59613,7 +59672,7 @@ static SDValue combineAddOfPMADDWD(SelectionDAG &DAG, SDValue N0, SDValue N1,
DAG.getVectorShuffle(OpVT, DL, N0.getOperand(0), N1.getOperand(0), Mask);
SDValue RHS =
DAG.getVectorShuffle(OpVT, DL, N0.getOperand(1), N1.getOperand(1), Mask);
- return DAG.getNode(X86ISD::VPMADDWD, DL, VT, LHS, RHS);
+ return DAG.getNode(N0.getOpcode(), DL, VT, LHS, RHS);
}
/// CMOV of constants requires materializing constant operands in registers.
@@ -59742,7 +59801,7 @@ static SDValue combineAdd(SDNode *N, SelectionDAG &DAG,
return MAdd;
if (SDValue MAdd = matchPMADDWD_2(DAG, N, DL, VT, Subtarget))
return MAdd;
- if (SDValue MAdd = combineAddOfPMADDWD(DAG, Op0, Op1, DL, VT))
+ if (SDValue MAdd = combineAddOfPMADD(DAG, Op0, Op1, DL, VT))
return MAdd;
// Try to synthesize horizontal adds from adds of shuffles.
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
index 082b3b60bb91a..37a4f475ebaba 100644
--- a/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
@@ -8,27 +8,13 @@ define <16 x i16> @combine_pmaddubsw_add_u8_s6(<32 x i8> %a0, <32 x i8> %a1) {
; AVX2-LABEL: combine_pmaddubsw_add_u8_s6:
; AVX2: # %bb.0:
; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
-; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
-; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
-; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpaddw %ymm0, %ymm2, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: combine_pmaddubsw_add_u8_s6:
; AVX512: # %bb.0:
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
-; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
-; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
-; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
-; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
-; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpaddw %ymm0, %ymm2, %ymm0
; AVX512-NEXT: retq
%and = and <32 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
%even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
@@ -46,27 +32,13 @@ define <16 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<32 x i8> %a0, <32 x i8>
; AVX2-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
; AVX2: # %bb.0:
; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX2-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX2-NEXT: vpand %ymm2, %ymm0, %ymm3
-; AVX2-NEXT: vpand %ymm2, %ymm1, %ymm2
-; AVX2-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
-; AVX2-NEXT: vpsrlw $8, %ymm0, %ymm0
-; AVX2-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX2-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
-; AVX2-NEXT: vpaddw %ymm2, %ymm0, %ymm0
; AVX2-NEXT: retq
;
; AVX512-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
; AVX512: # %bb.0:
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
-; AVX512-NEXT: vpbroadcastw {{.*#+}} ymm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX512-NEXT: vpand %ymm2, %ymm0, %ymm3
-; AVX512-NEXT: vpand %ymm2, %ymm1, %ymm2
-; AVX512-NEXT: vpmaddubsw %ymm2, %ymm3, %ymm2
-; AVX512-NEXT: vpsrlw $8, %ymm0, %ymm0
-; AVX512-NEXT: vpsrlw $8, %ymm1, %ymm1
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
-; AVX512-NEXT: vpaddw %ymm2, %ymm0, %ymm0
; AVX512-NEXT: retq
%and = and <32 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
%even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
index 54e21d41a8a49..4daf64f3c910c 100644
--- a/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
@@ -7,14 +7,7 @@ define <32 x i16> @combine_pmaddubsw_add_u8_s6(<64 x i8> %a0, <64 x i8> %a1) {
; AVX512-LABEL: combine_pmaddubsw_add_u8_s6:
; AVX512: # %bb.0:
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
-; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm3
-; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm2
-; AVX512-NEXT: vpmaddubsw %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512-NEXT: vpsrlw $8, %zmm1, %zmm1
; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpaddw %zmm0, %zmm2, %zmm0
; AVX512-NEXT: retq
%and = and <64 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
%even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
@@ -32,14 +25,7 @@ define <32 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<64 x i8> %a0, <64 x i8>
; AVX512-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
; AVX512: # %bb.0:
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
-; AVX512-NEXT: vpbroadcastw {{.*#+}} zmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX512-NEXT: vpandq %zmm2, %zmm0, %zmm3
-; AVX512-NEXT: vpandq %zmm2, %zmm1, %zmm2
-; AVX512-NEXT: vpmaddubsw %zmm2, %zmm3, %zmm2
-; AVX512-NEXT: vpsrlw $8, %zmm0, %zmm0
-; AVX512-NEXT: vpsrlw $8, %zmm1, %zmm1
; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
-; AVX512-NEXT: vpaddw %zmm2, %zmm0, %zmm0
; AVX512-NEXT: retq
%and = and <64 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
%even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
index d6849eefac8f9..61790d2d08a3b 100644
--- a/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
@@ -172,59 +172,17 @@ define i32 @combine_pmaddubsw_constant_sat() {
; 255 * 63 + 255 * 63 fits in signed i16.
; The unsigned input may have its high bit set; keep it as operand 0.
define <8 x i16> @combine_pmaddubsw_add_u8_s6(<16 x i8> %a0, <16 x i8> %a1) {
-; SSSE3-LABEL: combine_pmaddubsw_add_u8_s6:
-; SSSE3: # %bb.0:
-; SSSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSSE3-NEXT: movdqa %xmm0, %xmm3
-; SSSE3-NEXT: pand %xmm2, %xmm3
-; SSSE3-NEXT: pand %xmm1, %xmm2
-; SSSE3-NEXT: pmaddubsw %xmm2, %xmm3
-; SSSE3-NEXT: psrlw $8, %xmm0
-; SSSE3-NEXT: psrlw $8, %xmm1
-; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
-; SSSE3-NEXT: paddw %xmm3, %xmm0
-; SSSE3-NEXT: retq
-;
-; SSE41-LABEL: combine_pmaddubsw_add_u8_s6:
-; SSE41: # %bb.0:
-; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT: movdqa %xmm0, %xmm3
-; SSE41-NEXT: pand %xmm2, %xmm3
-; SSE41-NEXT: pand %xmm1, %xmm2
-; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
-; SSE41-NEXT: psrlw $8, %xmm0
-; SSE41-NEXT: psrlw $8, %xmm1
-; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
-; SSE41-NEXT: paddw %xmm3, %xmm0
-; SSE41-NEXT: retq
-;
-; AVX1-LABEL: combine_pmaddubsw_add_u8_s6:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
-; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm2
-; AVX1-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpaddw %xmm0, %xmm2, %xmm0
-; AVX1-NEXT: retq
+; SSE-LABEL: combine_pmaddubsw_add_u8_s6:
+; SSE: # %bb.0:
+; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE-NEXT: retq
;
-; AVX2-LABEL: combine_pmaddubsw_add_u8_s6:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
-; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm2
-; AVX2-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
-; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpaddw %xmm0, %xmm2, %xmm0
-; AVX2-NEXT: retq
+; AVX-LABEL: combine_pmaddubsw_add_u8_s6:
+; AVX: # %bb.0:
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
%and = and <16 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
%even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
%even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
@@ -238,59 +196,17 @@ define <8 x i16> @combine_pmaddubsw_add_u8_s6(<16 x i8> %a0, <16 x i8> %a1) {
; The ADD operands can be commuted without changing PMADDUBSW operand roles.
define <8 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<16 x i8> %a0, <16 x i8> %a1) {
-; SSSE3-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
-; SSSE3: # %bb.0:
-; SSSE3-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSSE3-NEXT: movdqa {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; SSSE3-NEXT: movdqa %xmm0, %xmm3
-; SSSE3-NEXT: pand %xmm2, %xmm3
-; SSSE3-NEXT: pand %xmm1, %xmm2
-; SSSE3-NEXT: pmaddubsw %xmm2, %xmm3
-; SSSE3-NEXT: psrlw $8, %xmm0
-; SSSE3-NEXT: psrlw $8, %xmm1
-; SSSE3-NEXT: pmaddubsw %xmm1, %xmm0
-; SSSE3-NEXT: paddw %xmm3, %xmm0
-; SSSE3-NEXT: retq
-;
-; SSE41-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
-; SSE41: # %bb.0:
-; SSE41-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
-; SSE41-NEXT: pmovzxbw {{.*#+}} xmm2 = [255,255,255,255,255,255,255,255]
-; SSE41-NEXT: movdqa %xmm0, %xmm3
-; SSE41-NEXT: pand %xmm2, %xmm3
-; SSE41-NEXT: pand %xmm1, %xmm2
-; SSE41-NEXT: pmaddubsw %xmm2, %xmm3
-; SSE41-NEXT: psrlw $8, %xmm0
-; SSE41-NEXT: psrlw $8, %xmm1
-; SSE41-NEXT: pmaddubsw %xmm1, %xmm0
-; SSE41-NEXT: paddw %xmm3, %xmm0
-; SSE41-NEXT: retq
-;
-; AVX1-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
-; AVX1: # %bb.0:
-; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX1-NEXT: vbroadcastss {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX1-NEXT: vpand %xmm2, %xmm0, %xmm3
-; AVX1-NEXT: vpand %xmm2, %xmm1, %xmm2
-; AVX1-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
-; AVX1-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX1-NEXT: vpsrlw $8, %xmm1, %xmm1
-; AVX1-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
-; AVX1-NEXT: vpaddw %xmm2, %xmm0, %xmm0
-; AVX1-NEXT: retq
+; SSE-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; SSE: # %bb.0:
+; SSE-NEXT: pand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1
+; SSE-NEXT: pmaddubsw %xmm1, %xmm0
+; SSE-NEXT: retq
;
-; AVX2-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
-; AVX2: # %bb.0:
-; AVX2-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
-; AVX2-NEXT: vpbroadcastw {{.*#+}} xmm2 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0]
-; AVX2-NEXT: vpand %xmm2, %xmm0, %xmm3
-; AVX2-NEXT: vpand %xmm2, %xmm1, %xmm2
-; AVX2-NEXT: vpmaddubsw %xmm2, %xmm3, %xmm2
-; AVX2-NEXT: vpsrlw $8, %xmm0, %xmm0
-; AVX2-NEXT: vpsrlw $8, %xmm1, %xmm1
-; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
-; AVX2-NEXT: vpaddw %xmm2, %xmm0, %xmm0
-; AVX2-NEXT: retq
+; AVX-LABEL: combine_pmaddubsw_add_u8_s6_commuted:
+; AVX: # %bb.0:
+; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
+; AVX-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
+; AVX-NEXT: retq
%and = and <16 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
%even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
%even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
>From d06b9bff62464719eefe44ff595e19c46759cb1c Mon Sep 17 00:00:00 2001
From: Jihyeon Jeong <jh.jeong129 at gmail.com>
Date: Fri, 2 Oct 2026 10:38:57 +0000
Subject: [PATCH 3/3] [X86] Simplify PMADDUBSW test masks
---
llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll | 17 +++++++++--------
llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll | 17 +++++++++--------
llvm/test/CodeGen/X86/combine-pmaddubsw.ll | 17 +++++++++--------
3 files changed, 27 insertions(+), 24 deletions(-)
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
index 37a4f475ebaba..93da70f4f3853 100644
--- a/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw-256.ll
@@ -16,7 +16,7 @@ define <16 x i16> @combine_pmaddubsw_add_u8_s6(<32 x i8> %a0, <32 x i8> %a1) {
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: retq
- %and = and <32 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %and = and <32 x i8> %a1, splat (i8 63)
%even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
%even1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
%odd0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
@@ -40,7 +40,7 @@ define <16 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<32 x i8> %a0, <32 x i8>
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to8}, %ymm1, %ymm1
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: retq
- %and = and <32 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %and = and <32 x i8> %a1, splat (i8 63)
%even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
%even1 = shufflevector <32 x i8> %and, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
%odd0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
@@ -83,7 +83,7 @@ define <16 x i16> @combine_pmaddubsw_add_positive_overflow(<32 x i8> %a0, <32 x
; AVX512-NEXT: vpaddw %ymm0, %ymm2, %ymm0
; AVX512-NEXT: retq
%and0 = and <32 x i8> %a0, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 -1, i8 -1>
- %and1 = and <32 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %and1 = and <32 x i8> %a1, splat (i8 127)
%even0 = shufflevector <32 x i8> %and0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
%even1 = shufflevector <32 x i8> %and1, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
%odd0 = shufflevector <32 x i8> %and0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
@@ -121,7 +121,7 @@ define <16 x i16> @combine_pmaddubsw_add_negative_overflow(<32 x i8> %a0, <32 x
; AVX512-NEXT: vpmaddubsw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: vpaddw %ymm0, %ymm2, %ymm0
; AVX512-NEXT: retq
- %or = or <32 x i8> %a1, <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>
+ %or = or <32 x i8> %a1, splat (i8 -128)
%even0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
%even1 = shufflevector <32 x i8> %or, <32 x i8> zeroinitializer, <32 x i32> <i32 0, i32 32, i32 2, i32 32, i32 4, i32 32, i32 6, i32 32, i32 8, i32 32, i32 10, i32 32, i32 12, i32 32, i32 14, i32 32, i32 16, i32 32, i32 18, i32 32, i32 20, i32 32, i32 22, i32 32, i32 24, i32 32, i32 26, i32 32, i32 28, i32 32, i32 30, i32 32>
%odd0 = shufflevector <32 x i8> %a0, <32 x i8> zeroinitializer, <32 x i32> <i32 1, i32 32, i32 3, i32 32, i32 5, i32 32, i32 7, i32 32, i32 9, i32 32, i32 11, i32 32, i32 13, i32 32, i32 15, i32 32, i32 17, i32 32, i32 19, i32 32, i32 21, i32 32, i32 23, i32 32, i32 25, i32 32, i32 27, i32 32, i32 29, i32 32, i32 31, i32 32>
@@ -160,10 +160,11 @@ define <16 x i16> @combine_pmaddubsw_add_independent_inputs(<32 x i8> %a0, <32 x
; AVX512-NEXT: vpmaddubsw %ymm2, %ymm1, %ymm1
; AVX512-NEXT: vpaddw %ymm1, %ymm0, %ymm0
; AVX512-NEXT: retq
- %and0 = and <32 x i8> %a0, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
- %and1 = and <32 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
- %and2 = and <32 x i8> %a2, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
- %and3 = and <32 x i8> %a3, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %mask = bitcast <16 x i16> splat (i16 127) to <32 x i8>
+ %and0 = and <32 x i8> %a0, %mask
+ %and1 = and <32 x i8> %a1, splat (i8 127)
+ %and2 = and <32 x i8> %a2, %mask
+ %and3 = and <32 x i8> %a3, splat (i8 127)
%m0 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %and0, <32 x i8> %and1)
%m1 = call <16 x i16> @llvm.x86.avx2.pmadd.ub.sw(<32 x i8> %and2, <32 x i8> %and3)
%res = add <16 x i16> %m0, %m1
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
index 4daf64f3c910c..7a6c0b6bea543 100644
--- a/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw-512.ll
@@ -9,7 +9,7 @@ define <32 x i16> @combine_pmaddubsw_add_u8_s6(<64 x i8> %a0, <64 x i8> %a1) {
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: retq
- %and = and <64 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %and = and <64 x i8> %a1, splat (i8 63)
%even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
%even1 = shufflevector <64 x i8> %and, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
%odd0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
@@ -27,7 +27,7 @@ define <32 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<64 x i8> %a0, <64 x i8>
; AVX512-NEXT: vpandd {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to16}, %zmm1, %zmm1
; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: retq
- %and = and <64 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %and = and <64 x i8> %a1, splat (i8 63)
%even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
%even1 = shufflevector <64 x i8> %and, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
%odd0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
@@ -56,7 +56,7 @@ define <32 x i16> @combine_pmaddubsw_add_positive_overflow(<64 x i8> %a0, <64 x
; AVX512-NEXT: vpaddw %zmm0, %zmm2, %zmm0
; AVX512-NEXT: retq
%and0 = and <64 x i8> %a0, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 -1, i8 -1>
- %and1 = and <64 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %and1 = and <64 x i8> %a1, splat (i8 127)
%even0 = shufflevector <64 x i8> %and0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
%even1 = shufflevector <64 x i8> %and1, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
%odd0 = shufflevector <64 x i8> %and0, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
@@ -81,7 +81,7 @@ define <32 x i16> @combine_pmaddubsw_add_negative_overflow(<64 x i8> %a0, <64 x
; AVX512-NEXT: vpmaddubsw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: vpaddw %zmm0, %zmm2, %zmm0
; AVX512-NEXT: retq
- %or = or <64 x i8> %a1, <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>
+ %or = or <64 x i8> %a1, splat (i8 -128)
%even0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
%even1 = shufflevector <64 x i8> %or, <64 x i8> zeroinitializer, <64 x i32> <i32 0, i32 64, i32 2, i32 64, i32 4, i32 64, i32 6, i32 64, i32 8, i32 64, i32 10, i32 64, i32 12, i32 64, i32 14, i32 64, i32 16, i32 64, i32 18, i32 64, i32 20, i32 64, i32 22, i32 64, i32 24, i32 64, i32 26, i32 64, i32 28, i32 64, i32 30, i32 64, i32 32, i32 64, i32 34, i32 64, i32 36, i32 64, i32 38, i32 64, i32 40, i32 64, i32 42, i32 64, i32 44, i32 64, i32 46, i32 64, i32 48, i32 64, i32 50, i32 64, i32 52, i32 64, i32 54, i32 64, i32 56, i32 64, i32 58, i32 64, i32 60, i32 64, i32 62, i32 64>
%odd0 = shufflevector <64 x i8> %a0, <64 x i8> zeroinitializer, <64 x i32> <i32 1, i32 64, i32 3, i32 64, i32 5, i32 64, i32 7, i32 64, i32 9, i32 64, i32 11, i32 64, i32 13, i32 64, i32 15, i32 64, i32 17, i32 64, i32 19, i32 64, i32 21, i32 64, i32 23, i32 64, i32 25, i32 64, i32 27, i32 64, i32 29, i32 64, i32 31, i32 64, i32 33, i32 64, i32 35, i32 64, i32 37, i32 64, i32 39, i32 64, i32 41, i32 64, i32 43, i32 64, i32 45, i32 64, i32 47, i32 64, i32 49, i32 64, i32 51, i32 64, i32 53, i32 64, i32 55, i32 64, i32 57, i32 64, i32 59, i32 64, i32 61, i32 64, i32 63, i32 64>
@@ -107,10 +107,11 @@ define <32 x i16> @combine_pmaddubsw_add_independent_inputs(<64 x i8> %a0, <64 x
; AVX512-NEXT: vpmaddubsw %zmm2, %zmm1, %zmm1
; AVX512-NEXT: vpaddw %zmm1, %zmm0, %zmm0
; AVX512-NEXT: retq
- %and0 = and <64 x i8> %a0, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
- %and1 = and <64 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
- %and2 = and <64 x i8> %a2, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
- %and3 = and <64 x i8> %a3, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %mask = bitcast <32 x i16> splat (i16 127) to <64 x i8>
+ %and0 = and <64 x i8> %a0, %mask
+ %and1 = and <64 x i8> %a1, splat (i8 127)
+ %and2 = and <64 x i8> %a2, %mask
+ %and3 = and <64 x i8> %a3, splat (i8 127)
%m0 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %and0, <64 x i8> %and1)
%m1 = call <32 x i16> @llvm.x86.avx512.pmaddubs.w.512(<64 x i8> %and2, <64 x i8> %and3)
%res = add <32 x i16> %m0, %m1
diff --git a/llvm/test/CodeGen/X86/combine-pmaddubsw.ll b/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
index 61790d2d08a3b..15428dcea5139 100644
--- a/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
+++ b/llvm/test/CodeGen/X86/combine-pmaddubsw.ll
@@ -183,7 +183,7 @@ define <8 x i16> @combine_pmaddubsw_add_u8_s6(<16 x i8> %a0, <16 x i8> %a1) {
; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
; AVX-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
- %and = and <16 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %and = and <16 x i8> %a1, splat (i8 63)
%even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
%even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
%odd0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
@@ -207,7 +207,7 @@ define <8 x i16> @combine_pmaddubsw_add_u8_s6_commuted(<16 x i8> %a0, <16 x i8>
; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm1, %xmm1
; AVX-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
; AVX-NEXT: retq
- %and = and <16 x i8> %a1, <i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63, i8 63>
+ %and = and <16 x i8> %a1, splat (i8 63)
%even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
%even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
%odd0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
@@ -273,7 +273,7 @@ define <8 x i16> @combine_pmaddubsw_add_positive_overflow(<16 x i8> %a0, <16 x i
; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpaddw %xmm0, %xmm2, %xmm0
; AVX2-NEXT: retq
- %and = and <16 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %and = and <16 x i8> %a1, splat (i8 127)
%even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
%even1 = shufflevector <16 x i8> %and, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
%odd0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
@@ -339,7 +339,7 @@ define <8 x i16> @combine_pmaddubsw_add_negative_overflow(<16 x i8> %a0, <16 x i
; AVX2-NEXT: vpmaddubsw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpaddw %xmm0, %xmm2, %xmm0
; AVX2-NEXT: retq
- %or = or <16 x i8> %a1, <i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128, i8 -128>
+ %or = or <16 x i8> %a1, splat (i8 -128)
%even0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
%even1 = shufflevector <16 x i8> %or, <16 x i8> zeroinitializer, <16 x i32> <i32 0, i32 16, i32 2, i32 16, i32 4, i32 16, i32 6, i32 16, i32 8, i32 16, i32 10, i32 16, i32 12, i32 16, i32 14, i32 16>
%odd0 = shufflevector <16 x i8> %a0, <16 x i8> zeroinitializer, <16 x i32> <i32 1, i32 16, i32 3, i32 16, i32 5, i32 16, i32 7, i32 16, i32 9, i32 16, i32 11, i32 16, i32 13, i32 16, i32 15, i32 16>
@@ -404,10 +404,11 @@ define <8 x i16> @combine_pmaddubsw_add_independent_inputs(<16 x i8> %a0, <16 x
; AVX2-NEXT: vpmaddubsw %xmm2, %xmm1, %xmm1
; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
; AVX2-NEXT: retq
- %and0 = and <16 x i8> %a0, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
- %and1 = and <16 x i8> %a1, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
- %and2 = and <16 x i8> %a2, <i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0, i8 127, i8 0>
- %and3 = and <16 x i8> %a3, <i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127, i8 127>
+ %mask = bitcast <8 x i16> splat (i16 127) to <16 x i8>
+ %and0 = and <16 x i8> %a0, %mask
+ %and1 = and <16 x i8> %a1, splat (i8 127)
+ %and2 = and <16 x i8> %a2, %mask
+ %and3 = and <16 x i8> %a3, splat (i8 127)
%m0 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %and0, <16 x i8> %and1)
%m1 = call <8 x i16> @llvm.x86.ssse3.pmadd.ub.sw.128(<16 x i8> %and2, <16 x i8> %and3)
%res = add <8 x i16> %m0, %m1
More information about the llvm-commits
mailing list