[llvm] 8112a25 - [X86][SSE] Add 'vectorized sum' test patterns
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Mon Jan 11 04:51:35 PST 2021
Author: Simon Pilgrim
Date: 2021-01-11T12:51:18Z
New Revision: 8112a2598ce180ab4cd106f154a71e813fc28d91
URL: https://github.com/llvm/llvm-project/commit/8112a2598ce180ab4cd106f154a71e813fc28d91
DIFF: https://github.com/llvm/llvm-project/commit/8112a2598ce180ab4cd106f154a71e813fc28d91.diff
LOG: [X86][SSE] Add 'vectorized sum' test patterns
These are often generated when building a vector from the reduction sums of independent vectors.
I've implemented some typical patterns from various v4f32/v4i32 based off current codegen emitted from the vectorizers, although these tests are more about tweaking some hadd style backend folds to handle whatever the vectorizers/vectorcombine throws at us...
Added:
llvm/test/CodeGen/X86/horizontal-sum.ll
Modified:
Removed:
################################################################################
diff --git a/llvm/test/CodeGen/X86/horizontal-sum.ll b/llvm/test/CodeGen/X86/horizontal-sum.ll
new file mode 100644
index 000000000000..47d44171d99a
--- /dev/null
+++ b/llvm/test/CodeGen/X86/horizontal-sum.ll
@@ -0,0 +1,1189 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3 | FileCheck %s --check-prefixes=SSSE3,SSSE3-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+ssse3,fast-hops | FileCheck %s --check-prefixes=SSSE3,SSSE3-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX1-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx,fast-hops | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX1-FAST
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX,AVX-SLOW,AVX2-SLOW
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx2,fast-hops | FileCheck %s --check-prefixes=AVX,AVX-FAST,AVX2-FAST
+
+; Vectorized Pairwise Sum Reductions
+; e.g.
+; inline STYPE sum(VTYPE x) {
+; return (x[0] + x[1]) + (x[2] + x[3]);
+; }
+;
+; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) {
+; return (VTYPE) { sum( A0 ), sum( A1 ), sum( A2 ), sum( A3 ) };
+; }
+
+define <4 x float> @pair_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {
+; SSSE3-SLOW-LABEL: pair_sum_v4f32_v4f32:
+; SSSE3-SLOW: # %bb.0:
+; SSSE3-SLOW-NEXT: haddps %xmm0, %xmm0
+; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; SSSE3-SLOW-NEXT: addps %xmm4, %xmm0
+; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm1
+; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; SSSE3-SLOW-NEXT: addps %xmm1, %xmm4
+; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSSE3-SLOW-NEXT: haddps %xmm2, %xmm2
+; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-SLOW-NEXT: addps %xmm2, %xmm1
+; SSSE3-SLOW-NEXT: haddps %xmm3, %xmm3
+; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm2 = xmm3[1,1,3,3]
+; SSSE3-SLOW-NEXT: addps %xmm3, %xmm2
+; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
+; SSSE3-SLOW-NEXT: retq
+;
+; SSSE3-FAST-LABEL: pair_sum_v4f32_v4f32:
+; SSSE3-FAST: # %bb.0:
+; SSSE3-FAST-NEXT: haddps %xmm0, %xmm0
+; SSSE3-FAST-NEXT: haddps %xmm1, %xmm1
+; SSSE3-FAST-NEXT: haddps %xmm1, %xmm0
+; SSSE3-FAST-NEXT: haddps %xmm2, %xmm2
+; SSSE3-FAST-NEXT: haddps %xmm3, %xmm3
+; SSSE3-FAST-NEXT: haddps %xmm2, %xmm3
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[2,0]
+; SSSE3-FAST-NEXT: retq
+;
+; AVX1-SLOW-LABEL: pair_sum_v4f32_v4f32:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; AVX1-SLOW-NEXT: vaddps %xmm4, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vhaddps %xmm1, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; AVX1-SLOW-NEXT: vaddps %xmm4, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vunpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm1
+; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX1-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm3, %xmm1
+; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX1-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX1-SLOW-NEXT: retq
+;
+; AVX1-FAST-LABEL: pair_sum_v4f32_v4f32:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm1
+; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,1]
+; AVX1-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm1
+; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX1-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pair_sum_v4f32_v4f32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vhaddps %xmm0, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; AVX2-SLOW-NEXT: vaddps %xmm4, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vhaddps %xmm1, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; AVX2-SLOW-NEXT: vaddps %xmm4, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm1
+; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3]
+; AVX2-SLOW-NEXT: vhaddps %xmm3, %xmm3, %xmm1
+; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm2 = xmm1[1,1,3,3]
+; AVX2-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX2-SLOW-NEXT: retq
+;
+; AVX2-FAST-LABEL: pair_sum_v4f32_v4f32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
+; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm1
+; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
+; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3]
+; AVX2-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm1
+; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
+; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX2-FAST-NEXT: retq
+ %5 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %6 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %7 = fadd <2 x float> %5, %6
+ %8 = shufflevector <2 x float> %7, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
+ %9 = fadd <2 x float> %7, %8
+ %10 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %11 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %12 = fadd <2 x float> %10, %11
+ %13 = shufflevector <2 x float> %12, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
+ %14 = fadd <2 x float> %12, %13
+ %15 = shufflevector <2 x float> %9, <2 x float> %14, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+ %16 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %17 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %18 = fadd <2 x float> %16, %17
+ %19 = shufflevector <2 x float> %18, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
+ %20 = fadd <2 x float> %18, %19
+ %21 = shufflevector <2 x float> %20, <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+ %22 = shufflevector <4 x float> %15, <4 x float> %21, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
+ %23 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %24 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %25 = fadd <2 x float> %23, %24
+ %26 = shufflevector <2 x float> %25, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
+ %27 = fadd <2 x float> %25, %26
+ %28 = shufflevector <2 x float> %27, <2 x float> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+ %29 = shufflevector <4 x float> %22, <4 x float> %28, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ ret <4 x float> %29
+}
+
+define <4 x i32> @pair_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {
+; SSSE3-SLOW-LABEL: pair_sum_v4i32_v4i32:
+; SSSE3-SLOW: # %bb.0:
+; SSSE3-SLOW-NEXT: phaddd %xmm0, %xmm0
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm0
+; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm1
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm4
+; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm4[0],xmm0[1],xmm4[1]
+; SSSE3-SLOW-NEXT: phaddd %xmm2, %xmm2
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
+; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm1
+; SSSE3-SLOW-NEXT: phaddd %xmm3, %xmm3
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm3[1,1,1,1]
+; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm2
+; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm1[0]
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
+; SSSE3-SLOW-NEXT: retq
+;
+; SSSE3-FAST-LABEL: pair_sum_v4i32_v4i32:
+; SSSE3-FAST: # %bb.0:
+; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm0
+; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm1
+; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm0
+; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm2
+; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm3
+; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm3
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm3[2,0]
+; SSSE3-FAST-NEXT: retq
+;
+; AVX1-SLOW-LABEL: pair_sum_v4i32_v4i32:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vphaddd %xmm1, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX1-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm1
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm3, %xmm1
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
+; AVX1-SLOW-NEXT: retq
+;
+; AVX1-FAST-LABEL: pair_sum_v4i32_v4i32:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX1-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm1
+; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm1
+; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
+; AVX1-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pair_sum_v4i32_v4i32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm1
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3]
+; AVX2-SLOW-NEXT: vphaddd %xmm3, %xmm3, %xmm1
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX2-SLOW-NEXT: retq
+;
+; AVX2-FAST-LABEL: pair_sum_v4i32_v4i32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1
+; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm1
+; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1
+; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,3]
+; AVX2-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm1
+; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1
+; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX2-FAST-NEXT: retq
+ %5 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %6 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+ %7 = add <2 x i32> %5, %6
+ %8 = shufflevector <2 x i32> %7, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
+ %9 = add <2 x i32> %7, %8
+ %10 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %11 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+ %12 = add <2 x i32> %10, %11
+ %13 = shufflevector <2 x i32> %12, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
+ %14 = add <2 x i32> %12, %13
+ %15 = shufflevector <2 x i32> %9, <2 x i32> %14, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+ %16 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %17 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+ %18 = add <2 x i32> %16, %17
+ %19 = shufflevector <2 x i32> %18, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
+ %20 = add <2 x i32> %18, %19
+ %21 = shufflevector <2 x i32> %20, <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+ %22 = shufflevector <4 x i32> %15, <4 x i32> %21, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
+ %23 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+ %25 = add <2 x i32> %23, %24
+ %26 = shufflevector <2 x i32> %25, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
+ %27 = add <2 x i32> %25, %26
+ %28 = shufflevector <2 x i32> %27, <2 x i32> poison, <4 x i32> <i32 0, i32 undef, i32 undef, i32 undef>
+ %29 = shufflevector <4 x i32> %22, <4 x i32> %28, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ ret <4 x i32> %29
+}
+
+define <8 x float> @pair_sum_v8f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3, <4 x float> %4, <4 x float> %5, <4 x float> %6, <4 x float> %7) {
+; SSSE3-SLOW-LABEL: pair_sum_v8f32_v4f32:
+; SSSE3-SLOW: # %bb.0:
+; SSSE3-SLOW-NEXT: haddps %xmm0, %xmm0
+; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm8 = xmm0[1,1,3,3]
+; SSSE3-SLOW-NEXT: addps %xmm8, %xmm0
+; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm1
+; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm8 = xmm1[1,1,3,3]
+; SSSE3-SLOW-NEXT: addps %xmm1, %xmm8
+; SSSE3-SLOW-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm8[0],xmm0[1],xmm8[1]
+; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm1
+; SSSE3-SLOW-NEXT: haddps %xmm3, %xmm1
+; SSSE3-SLOW-NEXT: haddps %xmm2, %xmm2
+; SSSE3-SLOW-NEXT: haddps %xmm3, %xmm3
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm2[1,0]
+; SSSE3-SLOW-NEXT: movaps %xmm5, %xmm2
+; SSSE3-SLOW-NEXT: haddps %xmm4, %xmm5
+; SSSE3-SLOW-NEXT: haddps %xmm4, %xmm4
+; SSSE3-SLOW-NEXT: haddps %xmm2, %xmm2
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm4[1,1]
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm2[2,0]
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[2,0]
+; SSSE3-SLOW-NEXT: addps %xmm3, %xmm1
+; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-SLOW-NEXT: haddps %xmm6, %xmm6
+; SSSE3-SLOW-NEXT: haddps %xmm7, %xmm7
+; SSSE3-SLOW-NEXT: haddps %xmm7, %xmm6
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm6[0,2]
+; SSSE3-SLOW-NEXT: retq
+;
+; SSSE3-FAST-LABEL: pair_sum_v8f32_v4f32:
+; SSSE3-FAST: # %bb.0:
+; SSSE3-FAST-NEXT: haddps %xmm0, %xmm0
+; SSSE3-FAST-NEXT: haddps %xmm1, %xmm1
+; SSSE3-FAST-NEXT: haddps %xmm1, %xmm0
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; SSSE3-FAST-NEXT: movaps %xmm2, %xmm1
+; SSSE3-FAST-NEXT: haddps %xmm3, %xmm1
+; SSSE3-FAST-NEXT: haddps %xmm2, %xmm2
+; SSSE3-FAST-NEXT: haddps %xmm3, %xmm3
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[1,0],xmm2[1,0]
+; SSSE3-FAST-NEXT: movaps %xmm5, %xmm2
+; SSSE3-FAST-NEXT: haddps %xmm4, %xmm5
+; SSSE3-FAST-NEXT: haddps %xmm4, %xmm4
+; SSSE3-FAST-NEXT: haddps %xmm2, %xmm2
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,1],xmm4[1,1]
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[2,0],xmm2[2,0]
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[0,2],xmm5[2,0]
+; SSSE3-FAST-NEXT: addps %xmm3, %xmm1
+; SSSE3-FAST-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; SSSE3-FAST-NEXT: haddps %xmm6, %xmm6
+; SSSE3-FAST-NEXT: haddps %xmm7, %xmm7
+; SSSE3-FAST-NEXT: haddps %xmm7, %xmm6
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,3],xmm6[0,2]
+; SSSE3-FAST-NEXT: retq
+;
+; AVX1-SLOW-LABEL: pair_sum_v8f32_v4f32:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vhaddps %xmm0, %xmm0, %xmm8
+; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm0 = xmm8[1,1,3,3]
+; AVX1-SLOW-NEXT: vaddps %xmm0, %xmm8, %xmm8
+; AVX1-SLOW-NEXT: vhaddps %xmm1, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; AVX1-SLOW-NEXT: vaddps %xmm0, %xmm1, %xmm0
+; AVX1-SLOW-NEXT: vunpcklps {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
+; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm1
+; AVX1-SLOW-NEXT: vhaddps %xmm2, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm3, %xmm3
+; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[1],xmm3[1],zero,zero
+; AVX1-SLOW-NEXT: vhaddps %xmm4, %xmm4, %xmm3
+; AVX1-SLOW-NEXT: vhaddps %xmm5, %xmm5, %xmm4
+; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,1]
+; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]
+; AVX1-SLOW-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[1,3]
+; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[1]
+; AVX1-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-SLOW-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX1-SLOW-NEXT: vhaddps %xmm6, %xmm6, %xmm2
+; AVX1-SLOW-NEXT: vhaddps %xmm7, %xmm7, %xmm3
+; AVX1-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-SLOW-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
+; AVX1-SLOW-NEXT: retq
+;
+; AVX1-FAST-LABEL: pair_sum_v8f32_v4f32:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm1
+; AVX1-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm2
+; AVX1-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm3
+; AVX1-FAST-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[1],xmm3[1],zero,zero
+; AVX1-FAST-NEXT: vhaddps %xmm4, %xmm4, %xmm3
+; AVX1-FAST-NEXT: vhaddps %xmm5, %xmm5, %xmm4
+; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm1[0,2],xmm3[0,1]
+; AVX1-FAST-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[0]
+; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm2 = xmm2[0,1],xmm3[1,3]
+; AVX1-FAST-NEXT: vinsertps {{.*#+}} xmm2 = xmm2[0,1,2],xmm4[1]
+; AVX1-FAST-NEXT: vaddps %xmm2, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,1]
+; AVX1-FAST-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX1-FAST-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX1-FAST-NEXT: vhaddps %xmm6, %xmm6, %xmm2
+; AVX1-FAST-NEXT: vhaddps %xmm7, %xmm7, %xmm3
+; AVX1-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm2
+; AVX1-FAST-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; AVX1-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX1-FAST-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
+; AVX1-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pair_sum_v8f32_v4f32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vhaddps %xmm0, %xmm0, %xmm8
+; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm0 = xmm8[1,1,3,3]
+; AVX2-SLOW-NEXT: vaddps %xmm0, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vhaddps %xmm1, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vmovshdup {{.*#+}} xmm0 = xmm1[1,1,3,3]
+; AVX2-SLOW-NEXT: vaddps %xmm0, %xmm1, %xmm0
+; AVX2-SLOW-NEXT: vunpcklps {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
+; AVX2-SLOW-NEXT: vhaddps %xmm4, %xmm4, %xmm1
+; AVX2-SLOW-NEXT: vhaddps %xmm5, %xmm5, %xmm4
+; AVX2-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm2
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3]
+; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
+; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1]
+; AVX2-SLOW-NEXT: vaddps %xmm1, %xmm3, %xmm1
+; AVX2-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-SLOW-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX2-SLOW-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX2-SLOW-NEXT: vhaddps %xmm6, %xmm6, %xmm2
+; AVX2-SLOW-NEXT: vhaddps %xmm7, %xmm7, %xmm3
+; AVX2-SLOW-NEXT: vhaddps %xmm3, %xmm2, %xmm2
+; AVX2-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; AVX2-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
+; AVX2-SLOW-NEXT: retq
+;
+; AVX2-FAST-LABEL: pair_sum_v8f32_v4f32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vhaddps %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm1, %xmm1
+; AVX2-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vhaddps %xmm4, %xmm4, %xmm1
+; AVX2-FAST-NEXT: vhaddps %xmm5, %xmm5, %xmm4
+; AVX2-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm2
+; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3]
+; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
+; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
+; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1]
+; AVX2-FAST-NEXT: vaddps %xmm1, %xmm3, %xmm1
+; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,1]
+; AVX2-FAST-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX2-FAST-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1
+; AVX2-FAST-NEXT: vhaddps %xmm6, %xmm6, %xmm2
+; AVX2-FAST-NEXT: vhaddps %xmm7, %xmm7, %xmm3
+; AVX2-FAST-NEXT: vhaddps %xmm3, %xmm2, %xmm2
+; AVX2-FAST-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; AVX2-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm0, %ymm0
+; AVX2-FAST-NEXT: vshufpd {{.*#+}} ymm0 = ymm1[0],ymm0[1],ymm1[2],ymm0[2]
+; AVX2-FAST-NEXT: retq
+ %9 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %10 = shufflevector <4 x float> %0, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %11 = fadd <2 x float> %9, %10
+ %12 = shufflevector <2 x float> %11, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
+ %13 = fadd <2 x float> %11, %12
+ %14 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %15 = shufflevector <4 x float> %1, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %16 = fadd <2 x float> %14, %15
+ %17 = shufflevector <2 x float> %16, <2 x float> poison, <2 x i32> <i32 1, i32 undef>
+ %18 = fadd <2 x float> %16, %17
+ %19 = shufflevector <2 x float> %13, <2 x float> %18, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ %20 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %21 = shufflevector <4 x float> %2, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %22 = fadd <2 x float> %20, %21
+ %23 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %24 = shufflevector <4 x float> %3, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %25 = fadd <2 x float> %23, %24
+ %26 = shufflevector <4 x float> %4, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %27 = shufflevector <4 x float> %4, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %28 = fadd <2 x float> %26, %27
+ %29 = shufflevector <2 x float> %28, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %30 = shufflevector <4 x float> %5, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %31 = shufflevector <4 x float> %5, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %32 = fadd <2 x float> %30, %31
+ %33 = shufflevector <2 x float> %32, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %34 = shufflevector <2 x float> %22, <2 x float> %25, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+ %35 = shufflevector <4 x float> %34, <4 x float> %29, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
+ %36 = shufflevector <4 x float> %35, <4 x float> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %37 = shufflevector <2 x float> %22, <2 x float> %25, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+ %38 = shufflevector <4 x float> %37, <4 x float> %29, <4 x i32> <i32 0, i32 1, i32 5, i32 undef>
+ %39 = shufflevector <4 x float> %38, <4 x float> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 5>
+ %40 = fadd <4 x float> %36, %39
+ %41 = shufflevector <4 x float> %40, <4 x float> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+ %42 = shufflevector <8 x float> %19, <8 x float> %41, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef>
+ %43 = shufflevector <4 x float> %6, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %44 = shufflevector <4 x float> %6, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %45 = fadd <2 x float> %43, %44
+ %46 = shufflevector <4 x float> %7, <4 x float> poison, <2 x i32> <i32 0, i32 2>
+ %47 = shufflevector <4 x float> %7, <4 x float> poison, <2 x i32> <i32 1, i32 3>
+ %48 = fadd <2 x float> %46, %47
+ %49 = shufflevector <2 x float> %45, <2 x float> %48, <2 x i32> <i32 0, i32 2>
+ %50 = shufflevector <2 x float> %45, <2 x float> %48, <2 x i32> <i32 1, i32 3>
+ %51 = fadd <2 x float> %49, %50
+ %52 = shufflevector <2 x float> %51, <2 x float> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ %53 = shufflevector <8 x float> %42, <8 x float> %52, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+ ret <8 x float> %53
+}
+
+define <8 x i32> @pair_sum_v8i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3, <4 x i32> %4, <4 x i32> %5, <4 x i32> %6, <4 x i32> %7) {
+; SSSE3-SLOW-LABEL: pair_sum_v8i32_v4i32:
+; SSSE3-SLOW: # %bb.0:
+; SSSE3-SLOW-NEXT: movdqa %xmm2, %xmm8
+; SSSE3-SLOW-NEXT: phaddd %xmm0, %xmm0
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm0[1,1,1,1]
+; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm0
+; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm1
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm2
+; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1]
+; SSSE3-SLOW-NEXT: phaddd %xmm3, %xmm8
+; SSSE3-SLOW-NEXT: movdqa %xmm5, %xmm1
+; SSSE3-SLOW-NEXT: phaddd %xmm4, %xmm5
+; SSSE3-SLOW-NEXT: phaddd %xmm4, %xmm4
+; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm1
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[1,1]
+; SSSE3-SLOW-NEXT: movdqa %xmm8, %xmm2
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[0,2],xmm5[2,0]
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm8 = xmm8[1,3],xmm1[2,0]
+; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm8
+; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm8[0]
+; SSSE3-SLOW-NEXT: phaddd %xmm6, %xmm6
+; SSSE3-SLOW-NEXT: phaddd %xmm7, %xmm7
+; SSSE3-SLOW-NEXT: phaddd %xmm7, %xmm6
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm8 = xmm8[2,3],xmm6[0,2]
+; SSSE3-SLOW-NEXT: movaps %xmm8, %xmm1
+; SSSE3-SLOW-NEXT: retq
+;
+; SSSE3-FAST-LABEL: pair_sum_v8i32_v4i32:
+; SSSE3-FAST: # %bb.0:
+; SSSE3-FAST-NEXT: phaddd %xmm0, %xmm0
+; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm1
+; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm0
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm2
+; SSSE3-FAST-NEXT: movdqa %xmm5, %xmm1
+; SSSE3-FAST-NEXT: phaddd %xmm4, %xmm5
+; SSSE3-FAST-NEXT: phaddd %xmm4, %xmm4
+; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm1
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[1,1],xmm4[1,1]
+; SSSE3-FAST-NEXT: movdqa %xmm2, %xmm3
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[0,2],xmm5[2,0]
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm2 = xmm2[1,3],xmm1[2,0]
+; SSSE3-FAST-NEXT: paddd %xmm3, %xmm2
+; SSSE3-FAST-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSSE3-FAST-NEXT: phaddd %xmm6, %xmm6
+; SSSE3-FAST-NEXT: phaddd %xmm7, %xmm7
+; SSSE3-FAST-NEXT: phaddd %xmm7, %xmm6
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm2 = xmm2[2,3],xmm6[0,2]
+; SSSE3-FAST-NEXT: movaps %xmm2, %xmm1
+; SSSE3-FAST-NEXT: retq
+;
+; AVX1-SLOW-LABEL: pair_sum_v8i32_v4i32:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vphaddd %xmm0, %xmm0, %xmm8
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm8, %xmm8
+; AVX1-SLOW-NEXT: vphaddd %xmm1, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm1, %xmm0
+; AVX1-SLOW-NEXT: vpunpckldq {{.*#+}} xmm8 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
+; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm2, %xmm1
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
+; AVX1-SLOW-NEXT: vphaddd %xmm2, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm3, %xmm3
+; AVX1-SLOW-NEXT: vphaddd %xmm4, %xmm4, %xmm4
+; AVX1-SLOW-NEXT: vphaddd %xmm5, %xmm5, %xmm5
+; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm5[0,0,0,0]
+; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
+; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]
+; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm4[1],zero
+; AVX1-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[1]
+; AVX1-SLOW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm8[0],xmm0[0]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-SLOW-NEXT: vphaddd %xmm6, %xmm6, %xmm2
+; AVX1-SLOW-NEXT: vphaddd %xmm7, %xmm7, %xmm3
+; AVX1-SLOW-NEXT: vphaddd %xmm3, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; AVX1-SLOW-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-SLOW-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2]
+; AVX1-SLOW-NEXT: retq
+;
+; AVX1-FAST-LABEL: pair_sum_v8i32_v4i32:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm8 = xmm0[0,2,1,3]
+; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm1
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
+; AVX1-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm2
+; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm3
+; AVX1-FAST-NEXT: vphaddd %xmm4, %xmm4, %xmm4
+; AVX1-FAST-NEXT: vphaddd %xmm5, %xmm5, %xmm5
+; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm1[0],xmm4[0]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm5[0,0,0,0]
+; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3,4,5],xmm0[6,7]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
+; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm3[2,3],xmm1[4,5,6,7]
+; AVX1-FAST-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1],xmm4[1],zero
+; AVX1-FAST-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm5[3]
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm1 = xmm8[0],xmm0[0]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3]
+; AVX1-FAST-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0
+; AVX1-FAST-NEXT: vphaddd %xmm6, %xmm6, %xmm2
+; AVX1-FAST-NEXT: vphaddd %xmm7, %xmm7, %xmm3
+; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm2
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,2,2,3]
+; AVX1-FAST-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1
+; AVX1-FAST-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2]
+; AVX1-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: pair_sum_v8i32_v4i32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vphaddd %xmm0, %xmm0, %xmm8
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm8[1,1,1,1]
+; AVX2-SLOW-NEXT: vpaddd %xmm0, %xmm8, %xmm8
+; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; AVX2-SLOW-NEXT: vpaddd %xmm0, %xmm1, %xmm0
+; AVX2-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm8[0],xmm0[0],xmm8[1],xmm0[1]
+; AVX2-SLOW-NEXT: vphaddd %xmm4, %xmm4, %xmm1
+; AVX2-SLOW-NEXT: vphaddd %xmm5, %xmm5, %xmm4
+; AVX2-SLOW-NEXT: vphaddd %xmm3, %xmm2, %xmm2
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3]
+; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
+; AVX2-SLOW-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
+; AVX2-SLOW-NEXT: vinsertps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[1]
+; AVX2-SLOW-NEXT: vpaddd %xmm1, %xmm3, %xmm1
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX2-SLOW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-SLOW-NEXT: vphaddd %xmm6, %xmm6, %xmm1
+; AVX2-SLOW-NEXT: vphaddd %xmm7, %xmm7, %xmm2
+; AVX2-SLOW-NEXT: vphaddd %xmm2, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-SLOW-NEXT: vpbroadcastq %xmm1, %ymm1
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-SLOW-NEXT: retq
+;
+; AVX2-FAST-LABEL: pair_sum_v8i32_v4i32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vphaddd %xmm0, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm1, %xmm1
+; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX2-FAST-NEXT: vphaddd %xmm4, %xmm4, %xmm1
+; AVX2-FAST-NEXT: vphaddd %xmm5, %xmm5, %xmm4
+; AVX2-FAST-NEXT: vphaddd %xmm3, %xmm2, %xmm2
+; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm3 = xmm2[0,2],xmm1[0,3]
+; AVX2-FAST-NEXT: vinsertps {{.*#+}} xmm3 = xmm3[0,1,2],xmm4[0]
+; AVX2-FAST-NEXT: vshufps {{.*#+}} xmm1 = xmm2[1,3],xmm1[1,3]
+; AVX2-FAST-NEXT: vblendps {{.*#+}} xmm1 = xmm1[0,1,2],xmm4[3]
+; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm3, %xmm1
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX2-FAST-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm0
+; AVX2-FAST-NEXT: vphaddd %xmm6, %xmm6, %xmm1
+; AVX2-FAST-NEXT: vphaddd %xmm7, %xmm7, %xmm2
+; AVX2-FAST-NEXT: vphaddd %xmm2, %xmm1, %xmm1
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,2,3]
+; AVX2-FAST-NEXT: vpbroadcastq %xmm1, %ymm1
+; AVX2-FAST-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1,2,3,4,5],ymm1[6,7]
+; AVX2-FAST-NEXT: retq
+ %9 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %10 = shufflevector <4 x i32> %0, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+ %11 = add <2 x i32> %9, %10
+ %12 = shufflevector <2 x i32> %11, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
+ %13 = add <2 x i32> %11, %12
+ %14 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %15 = shufflevector <4 x i32> %1, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+ %16 = add <2 x i32> %14, %15
+ %17 = shufflevector <2 x i32> %16, <2 x i32> poison, <2 x i32> <i32 1, i32 undef>
+ %18 = add <2 x i32> %16, %17
+ %19 = shufflevector <2 x i32> %13, <2 x i32> %18, <8 x i32> <i32 0, i32 2, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ %20 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %21 = shufflevector <4 x i32> %2, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+ %22 = add <2 x i32> %20, %21
+ %23 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+ %25 = add <2 x i32> %23, %24
+ %26 = shufflevector <4 x i32> %4, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %27 = shufflevector <4 x i32> %4, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+ %28 = add <2 x i32> %26, %27
+ %29 = shufflevector <2 x i32> %28, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %30 = shufflevector <4 x i32> %5, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %31 = shufflevector <4 x i32> %5, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+ %32 = add <2 x i32> %30, %31
+ %33 = shufflevector <2 x i32> %32, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %34 = shufflevector <2 x i32> %22, <2 x i32> %25, <4 x i32> <i32 0, i32 2, i32 undef, i32 undef>
+ %35 = shufflevector <4 x i32> %34, <4 x i32> %29, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
+ %36 = shufflevector <4 x i32> %35, <4 x i32> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ %37 = shufflevector <2 x i32> %22, <2 x i32> %25, <4 x i32> <i32 1, i32 3, i32 undef, i32 undef>
+ %38 = shufflevector <4 x i32> %37, <4 x i32> %29, <4 x i32> <i32 0, i32 1, i32 5, i32 undef>
+ %39 = shufflevector <4 x i32> %38, <4 x i32> %33, <4 x i32> <i32 0, i32 1, i32 2, i32 5>
+ %40 = add <4 x i32> %36, %39
+ %41 = shufflevector <4 x i32> %40, <4 x i32> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 undef, i32 undef, i32 undef, i32 undef>
+ %42 = shufflevector <8 x i32> %19, <8 x i32> %41, <8 x i32> <i32 0, i32 1, i32 8, i32 9, i32 10, i32 11, i32 undef, i32 undef>
+ %43 = shufflevector <4 x i32> %6, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %44 = shufflevector <4 x i32> %6, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+ %45 = add <2 x i32> %43, %44
+ %46 = shufflevector <4 x i32> %7, <4 x i32> poison, <2 x i32> <i32 0, i32 2>
+ %47 = shufflevector <4 x i32> %7, <4 x i32> poison, <2 x i32> <i32 1, i32 3>
+ %48 = add <2 x i32> %46, %47
+ %49 = shufflevector <2 x i32> %45, <2 x i32> %48, <2 x i32> <i32 0, i32 2>
+ %50 = shufflevector <2 x i32> %45, <2 x i32> %48, <2 x i32> <i32 1, i32 3>
+ %51 = add <2 x i32> %49, %50
+ %52 = shufflevector <2 x i32> %51, <2 x i32> poison, <8 x i32> <i32 0, i32 1, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef, i32 undef>
+ %53 = shufflevector <8 x i32> %42, <8 x i32> %52, <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 8, i32 9>
+ ret <8 x i32> %53
+}
+
+; Vectorized Sequential Sum Reductions
+; e.g.
+; inline STYPE sum(VTYPE x) {
+; return ((x[0] + x[1]) + x[2]) + x[3];
+; }
+;
+; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) {
+; return (VTYPE) { sum( A0 ), sum( A1 ), sum( A2 ), sum( A3 ) };
+; }
+
+define <4 x float> @sequential_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {
+; SSSE3-SLOW-LABEL: sequential_sum_v4f32_v4f32:
+; SSSE3-SLOW: # %bb.0:
+; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm4
+; SSSE3-SLOW-NEXT: movaps %xmm0, %xmm5
+; SSSE3-SLOW-NEXT: haddps %xmm1, %xmm5
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,2,2,3]
+; SSSE3-SLOW-NEXT: unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-SLOW-NEXT: addps %xmm5, %xmm0
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3],xmm4[3,3]
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0,2,3]
+; SSSE3-SLOW-NEXT: addps %xmm1, %xmm0
+; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-SLOW-NEXT: addps %xmm2, %xmm1
+; SSSE3-SLOW-NEXT: movaps %xmm2, %xmm4
+; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]
+; SSSE3-SLOW-NEXT: addps %xmm1, %xmm4
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; SSSE3-SLOW-NEXT: addps %xmm4, %xmm2
+; SSSE3-SLOW-NEXT: movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; SSSE3-SLOW-NEXT: addps %xmm3, %xmm1
+; SSSE3-SLOW-NEXT: movaps %xmm3, %xmm4
+; SSSE3-SLOW-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; SSSE3-SLOW-NEXT: addps %xmm1, %xmm4
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSSE3-SLOW-NEXT: addps %xmm4, %xmm3
+; SSSE3-SLOW-NEXT: movlhps {{.*#+}} xmm3 = xmm3[0],xmm2[0]
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
+; SSSE3-SLOW-NEXT: retq
+;
+; SSSE3-FAST-LABEL: sequential_sum_v4f32_v4f32:
+; SSSE3-FAST: # %bb.0:
+; SSSE3-FAST-NEXT: movaps %xmm0, %xmm4
+; SSSE3-FAST-NEXT: movaps %xmm0, %xmm5
+; SSSE3-FAST-NEXT: haddps %xmm1, %xmm5
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm5 = xmm5[0,2,2,3]
+; SSSE3-FAST-NEXT: unpckhps {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-FAST-NEXT: addps %xmm5, %xmm0
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3],xmm4[3,3]
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm1 = xmm1[2,0,2,3]
+; SSSE3-FAST-NEXT: addps %xmm1, %xmm0
+; SSSE3-FAST-NEXT: movaps %xmm2, %xmm1
+; SSSE3-FAST-NEXT: haddps %xmm2, %xmm1
+; SSSE3-FAST-NEXT: movaps %xmm2, %xmm4
+; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm2[1]
+; SSSE3-FAST-NEXT: addps %xmm1, %xmm4
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; SSSE3-FAST-NEXT: addps %xmm4, %xmm2
+; SSSE3-FAST-NEXT: movaps %xmm3, %xmm1
+; SSSE3-FAST-NEXT: haddps %xmm3, %xmm1
+; SSSE3-FAST-NEXT: movaps %xmm3, %xmm4
+; SSSE3-FAST-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; SSSE3-FAST-NEXT: addps %xmm1, %xmm4
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSSE3-FAST-NEXT: addps %xmm4, %xmm3
+; SSSE3-FAST-NEXT: movlhps {{.*#+}} xmm3 = xmm3[0],xmm2[0]
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
+; SSSE3-FAST-NEXT: retq
+;
+; AVX-SLOW-LABEL: sequential_sum_v4f32_v4f32:
+; AVX-SLOW: # %bb.0:
+; AVX-SLOW-NEXT: vhaddps %xmm1, %xmm0, %xmm4
+; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; AVX-SLOW-NEXT: vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX-SLOW-NEXT: vaddps %xmm4, %xmm5, %xmm4
+; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
+; AVX-SLOW-NEXT: vaddps %xmm4, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; AVX-SLOW-NEXT: vaddps %xmm2, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
+; AVX-SLOW-NEXT: vaddps %xmm1, %xmm4, %xmm1
+; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX-SLOW-NEXT: vaddps %xmm1, %xmm2, %xmm1
+; AVX-SLOW-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-SLOW-NEXT: vmovshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; AVX-SLOW-NEXT: vaddps %xmm3, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
+; AVX-SLOW-NEXT: vaddps %xmm1, %xmm2, %xmm1
+; AVX-SLOW-NEXT: vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3]
+; AVX-SLOW-NEXT: vaddps %xmm1, %xmm2, %xmm1
+; AVX-SLOW-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX-SLOW-NEXT: retq
+;
+; AVX-FAST-LABEL: sequential_sum_v4f32_v4f32:
+; AVX-FAST: # %bb.0:
+; AVX-FAST-NEXT: vhaddps %xmm1, %xmm0, %xmm4
+; AVX-FAST-NEXT: vpermilps {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; AVX-FAST-NEXT: vunpckhps {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX-FAST-NEXT: vaddps %xmm4, %xmm5, %xmm4
+; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm1 = xmm1[1,0]
+; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[3],xmm1[1],zero,zero
+; AVX-FAST-NEXT: vaddps %xmm4, %xmm0, %xmm0
+; AVX-FAST-NEXT: vhaddps %xmm2, %xmm2, %xmm1
+; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
+; AVX-FAST-NEXT: vaddps %xmm1, %xmm4, %xmm1
+; AVX-FAST-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm1
+; AVX-FAST-NEXT: vmovlhps {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-FAST-NEXT: vhaddps %xmm3, %xmm3, %xmm1
+; AVX-FAST-NEXT: vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
+; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm1
+; AVX-FAST-NEXT: vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3]
+; AVX-FAST-NEXT: vaddps %xmm1, %xmm2, %xmm1
+; AVX-FAST-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX-FAST-NEXT: retq
+ %5 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 0, i32 4>
+ %6 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 1, i32 5>
+ %7 = fadd <2 x float> %5, %6
+ %8 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 2, i32 6>
+ %9 = fadd <2 x float> %8, %7
+ %10 = shufflevector <4 x float> %0, <4 x float> %1, <2 x i32> <i32 3, i32 7>
+ %11 = fadd <2 x float> %10, %9
+ %12 = shufflevector <2 x float> %11, <2 x float> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %13 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+ %14 = fadd <4 x float> %13, %2
+ %15 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+ %16 = fadd <4 x float> %15, %14
+ %17 = shufflevector <4 x float> %2, <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+ %18 = fadd <4 x float> %17, %16
+ %19 = shufflevector <4 x float> %12, <4 x float> %18, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
+ %20 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+ %21 = fadd <4 x float> %20, %3
+ %22 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+ %23 = fadd <4 x float> %22, %21
+ %24 = shufflevector <4 x float> %3, <4 x float> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+ %25 = fadd <4 x float> %24, %23
+ %26 = shufflevector <4 x float> %19, <4 x float> %25, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ ret <4 x float> %26
+}
+
+define <4 x i32> @sequential_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {
+; SSSE3-SLOW-LABEL: sequential_sum_v4i32_v4i32:
+; SSSE3-SLOW: # %bb.0:
+; SSSE3-SLOW-NEXT: movdqa %xmm0, %xmm4
+; SSSE3-SLOW-NEXT: phaddd %xmm1, %xmm4
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; SSSE3-SLOW-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm1
+; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm0
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm4
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm2
+; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm2
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm3[2,3,2,3]
+; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm4
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm3
+; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm3
+; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm3 = xmm3[0],xmm2[0]
+; SSSE3-SLOW-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,0]
+; SSSE3-SLOW-NEXT: retq
+;
+; SSSE3-FAST-LABEL: sequential_sum_v4i32_v4i32:
+; SSSE3-FAST: # %bb.0:
+; SSSE3-FAST-NEXT: movdqa %xmm0, %xmm4
+; SSSE3-FAST-NEXT: phaddd %xmm1, %xmm4
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; SSSE3-FAST-NEXT: punpckhdq {{.*#+}} xmm0 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSSE3-FAST-NEXT: paddd %xmm4, %xmm1
+; SSSE3-FAST-NEXT: paddd %xmm1, %xmm0
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm2[3,3,3,3]
+; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm2
+; SSSE3-FAST-NEXT: paddd %xmm2, %xmm4
+; SSSE3-FAST-NEXT: paddd %xmm1, %xmm4
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm2 = xmm3[3,3,3,3]
+; SSSE3-FAST-NEXT: phaddd %xmm3, %xmm3
+; SSSE3-FAST-NEXT: paddd %xmm3, %xmm2
+; SSSE3-FAST-NEXT: paddd %xmm1, %xmm2
+; SSSE3-FAST-NEXT: punpcklqdq {{.*#+}} xmm2 = xmm2[0],xmm4[0]
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,0]
+; SSSE3-FAST-NEXT: retq
+;
+; AVX1-SLOW-LABEL: sequential_sum_v4i32_v4i32:
+; AVX1-SLOW: # %bb.0:
+; AVX1-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm4
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; AVX1-SLOW-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm4, %xmm0
+; AVX1-SLOW-NEXT: vpaddd %xmm0, %xmm5, %xmm0
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddd %xmm4, %xmm2, %xmm4
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm4, %xmm2
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX1-SLOW-NEXT: vpaddd %xmm3, %xmm2, %xmm2
+; AVX1-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX1-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; AVX1-SLOW-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
+; AVX1-SLOW-NEXT: retq
+;
+; AVX1-FAST-LABEL: sequential_sum_v4i32_v4i32:
+; AVX1-FAST: # %bb.0:
+; AVX1-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm4
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; AVX1-FAST-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5,6,7]
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm4, %xmm0
+; AVX1-FAST-NEXT: vpaddd %xmm0, %xmm5, %xmm0
+; AVX1-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm1
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX1-FAST-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm4, %xmm1
+; AVX1-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX1-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm1
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX1-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm1
+; AVX1-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1
+; AVX1-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,0,0,0]
+; AVX1-FAST-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1,2,3,4,5],xmm1[6,7]
+; AVX1-FAST-NEXT: retq
+;
+; AVX2-SLOW-LABEL: sequential_sum_v4i32_v4i32:
+; AVX2-SLOW: # %bb.0:
+; AVX2-SLOW-NEXT: vphaddd %xmm1, %xmm0, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; AVX2-SLOW-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX2-SLOW-NEXT: vpaddd %xmm0, %xmm4, %xmm0
+; AVX2-SLOW-NEXT: vpaddd %xmm0, %xmm5, %xmm0
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[1,1,1,1]
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; AVX2-SLOW-NEXT: vpaddd %xmm4, %xmm2, %xmm4
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm4, %xmm2
+; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm3[1,1,1,1]
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
+; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX2-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX2-SLOW-NEXT: vpaddd %xmm3, %xmm2, %xmm2
+; AVX2-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpbroadcastd %xmm1, %xmm1
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
+; AVX2-SLOW-NEXT: retq
+;
+; AVX2-FAST-LABEL: sequential_sum_v4i32_v4i32:
+; AVX2-FAST: # %bb.0:
+; AVX2-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm4
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm4[0,2,2,3]
+; AVX2-FAST-NEXT: vpunpckhdq {{.*#+}} xmm5 = xmm0[2],xmm1[2],xmm0[3],xmm1[3]
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[2,3,2,3]
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0],xmm1[1],xmm0[2,3]
+; AVX2-FAST-NEXT: vpaddd %xmm0, %xmm4, %xmm0
+; AVX2-FAST-NEXT: vpaddd %xmm0, %xmm5, %xmm0
+; AVX2-FAST-NEXT: vphaddd %xmm2, %xmm2, %xmm1
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm2[2,3,2,3]
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX2-FAST-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm4, %xmm1
+; AVX2-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX2-FAST-NEXT: vphaddd %xmm3, %xmm3, %xmm1
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
+; AVX2-FAST-NEXT: vpshufd {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; AVX2-FAST-NEXT: vpaddd %xmm3, %xmm1, %xmm1
+; AVX2-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1
+; AVX2-FAST-NEXT: vpbroadcastd %xmm1, %xmm1
+; AVX2-FAST-NEXT: vpblendd {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[3]
+; AVX2-FAST-NEXT: retq
+ %5 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 0, i32 4>
+ %6 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 1, i32 5>
+ %7 = add <2 x i32> %5, %6
+ %8 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 2, i32 6>
+ %9 = add <2 x i32> %8, %7
+ %10 = shufflevector <4 x i32> %0, <4 x i32> %1, <2 x i32> <i32 3, i32 7>
+ %11 = add <2 x i32> %10, %9
+ %12 = shufflevector <2 x i32> %11, <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 undef, i32 undef>
+ %13 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+ %14 = add <4 x i32> %13, %2
+ %15 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+ %16 = add <4 x i32> %15, %14
+ %17 = shufflevector <4 x i32> %2, <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+ %18 = add <4 x i32> %17, %16
+ %19 = shufflevector <4 x i32> %12, <4 x i32> %18, <4 x i32> <i32 0, i32 1, i32 4, i32 undef>
+ %20 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 1, i32 undef, i32 undef, i32 undef>
+ %21 = add <4 x i32> %20, %3
+ %22 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 2, i32 undef, i32 undef, i32 undef>
+ %23 = add <4 x i32> %22, %21
+ %24 = shufflevector <4 x i32> %3, <4 x i32> poison, <4 x i32> <i32 3, i32 undef, i32 undef, i32 undef>
+ %25 = add <4 x i32> %24, %23
+ %26 = shufflevector <4 x i32> %19, <4 x i32> %25, <4 x i32> <i32 0, i32 1, i32 2, i32 4>
+ ret <4 x i32> %26
+}
+
+; Vectorized Reductions
+; e.g.
+; VTYPE sum4(VTYPE A0, VTYPE A1, VTYPE A2, VTYPE A3) {
+; return (VTYPE) { reduce( A0 ), reduce( A1 ), reduce( A2 ), reduce( A3 ) };
+; }
+
+define <4 x float> @reduction_sum_v4f32_v4f32(<4 x float> %0, <4 x float> %1, <4 x float> %2, <4 x float> %3) {
+; SSSE3-LABEL: reduction_sum_v4f32_v4f32:
+; SSSE3: # %bb.0:
+; SSSE3-NEXT: movshdup {{.*#+}} xmm5 = xmm0[1,1,3,3]
+; SSSE3-NEXT: movss {{.*#+}} xmm4 = mem[0],zero,zero,zero
+; SSSE3-NEXT: addss %xmm4, %xmm5
+; SSSE3-NEXT: movaps %xmm0, %xmm6
+; SSSE3-NEXT: unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm0[1]
+; SSSE3-NEXT: addss %xmm5, %xmm6
+; SSSE3-NEXT: shufps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; SSSE3-NEXT: addss %xmm6, %xmm0
+; SSSE3-NEXT: movshdup {{.*#+}} xmm5 = xmm1[1,1,3,3]
+; SSSE3-NEXT: addss %xmm4, %xmm5
+; SSSE3-NEXT: movaps %xmm1, %xmm6
+; SSSE3-NEXT: unpckhpd {{.*#+}} xmm6 = xmm6[1],xmm1[1]
+; SSSE3-NEXT: addss %xmm5, %xmm6
+; SSSE3-NEXT: shufps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; SSSE3-NEXT: addss %xmm6, %xmm1
+; SSSE3-NEXT: unpcklps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-NEXT: movshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; SSSE3-NEXT: addss %xmm4, %xmm1
+; SSSE3-NEXT: movaps %xmm2, %xmm5
+; SSSE3-NEXT: unpckhpd {{.*#+}} xmm5 = xmm5[1],xmm2[1]
+; SSSE3-NEXT: addss %xmm1, %xmm5
+; SSSE3-NEXT: shufps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; SSSE3-NEXT: addss %xmm5, %xmm2
+; SSSE3-NEXT: movshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; SSSE3-NEXT: addss %xmm4, %xmm1
+; SSSE3-NEXT: movaps %xmm3, %xmm4
+; SSSE3-NEXT: unpckhpd {{.*#+}} xmm4 = xmm4[1],xmm3[1]
+; SSSE3-NEXT: addss %xmm1, %xmm4
+; SSSE3-NEXT: shufps {{.*#+}} xmm3 = xmm3[3,3,3,3]
+; SSSE3-NEXT: addss %xmm4, %xmm3
+; SSSE3-NEXT: unpcklps {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSSE3-NEXT: movlhps {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSSE3-NEXT: retq
+;
+; AVX-LABEL: reduction_sum_v4f32_v4f32:
+; AVX: # %bb.0:
+; AVX-NEXT: vmovshdup {{.*#+}} xmm4 = xmm0[1,1,3,3]
+; AVX-NEXT: vmovss {{.*#+}} xmm5 = mem[0],zero,zero,zero
+; AVX-NEXT: vaddss %xmm5, %xmm4, %xmm4
+; AVX-NEXT: vpermilpd {{.*#+}} xmm6 = xmm0[1,0]
+; AVX-NEXT: vaddss %xmm6, %xmm4, %xmm4
+; AVX-NEXT: vpermilps {{.*#+}} xmm0 = xmm0[3,3,3,3]
+; AVX-NEXT: vaddss %xmm0, %xmm4, %xmm0
+; AVX-NEXT: vmovshdup {{.*#+}} xmm4 = xmm1[1,1,3,3]
+; AVX-NEXT: vaddss %xmm5, %xmm4, %xmm4
+; AVX-NEXT: vpermilpd {{.*#+}} xmm6 = xmm1[1,0]
+; AVX-NEXT: vaddss %xmm6, %xmm4, %xmm4
+; AVX-NEXT: vpermilps {{.*#+}} xmm1 = xmm1[3,3,3,3]
+; AVX-NEXT: vaddss %xmm1, %xmm4, %xmm1
+; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[2,3]
+; AVX-NEXT: vmovshdup {{.*#+}} xmm1 = xmm2[1,1,3,3]
+; AVX-NEXT: vaddss %xmm5, %xmm1, %xmm1
+; AVX-NEXT: vpermilpd {{.*#+}} xmm4 = xmm2[1,0]
+; AVX-NEXT: vaddss %xmm4, %xmm1, %xmm1
+; AVX-NEXT: vpermilps {{.*#+}} xmm2 = xmm2[3,3,3,3]
+; AVX-NEXT: vaddss %xmm2, %xmm1, %xmm1
+; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1],xmm1[0],xmm0[3]
+; AVX-NEXT: vmovshdup {{.*#+}} xmm1 = xmm3[1,1,3,3]
+; AVX-NEXT: vaddss %xmm5, %xmm1, %xmm1
+; AVX-NEXT: vpermilpd {{.*#+}} xmm2 = xmm3[1,0]
+; AVX-NEXT: vaddss %xmm2, %xmm1, %xmm1
+; AVX-NEXT: vpermilps {{.*#+}} xmm2 = xmm3[3,3,3,3]
+; AVX-NEXT: vaddss %xmm2, %xmm1, %xmm1
+; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[0,1,2],xmm1[0]
+; AVX-NEXT: retq
+ %5 = call float @llvm.vector.reduce.fadd.f32.v4f32(float undef, <4 x float> %0)
+ %6 = call float @llvm.vector.reduce.fadd.f32.v4f32(float undef, <4 x float> %1)
+ %7 = call float @llvm.vector.reduce.fadd.f32.v4f32(float undef, <4 x float> %2)
+ %8 = call float @llvm.vector.reduce.fadd.f32.v4f32(float undef, <4 x float> %3)
+ %9 = insertelement <4 x float> undef, float %5, i32 0
+ %10 = insertelement <4 x float> %9, float %6, i32 1
+ %11 = insertelement <4 x float> %10, float %7, i32 2
+ %12 = insertelement <4 x float> %11, float %8, i32 3
+ ret <4 x float> %12
+}
+declare float @llvm.vector.reduce.fadd.f32.v4f32(float, <4 x float>)
+
+define <4 x i32> @reduction_sum_v4i32_v4i32(<4 x i32> %0, <4 x i32> %1, <4 x i32> %2, <4 x i32> %3) {
+; SSSE3-SLOW-LABEL: reduction_sum_v4i32_v4i32:
+; SSSE3-SLOW: # %bb.0:
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSSE3-SLOW-NEXT: paddd %xmm0, %xmm4
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm0 = xmm4[1,1,1,1]
+; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm0
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm4
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm4[1,1,1,1]
+; SSSE3-SLOW-NEXT: paddd %xmm4, %xmm1
+; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; SSSE3-SLOW-NEXT: paddd %xmm2, %xmm1
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm2
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm1 = xmm3[2,3,2,3]
+; SSSE3-SLOW-NEXT: paddd %xmm3, %xmm1
+; SSSE3-SLOW-NEXT: pshufd {{.*#+}} xmm3 = xmm1[1,1,1,1]
+; SSSE3-SLOW-NEXT: paddd %xmm1, %xmm3
+; SSSE3-SLOW-NEXT: punpckldq {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1]
+; SSSE3-SLOW-NEXT: punpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm2[0]
+; SSSE3-SLOW-NEXT: retq
+;
+; SSSE3-FAST-LABEL: reduction_sum_v4i32_v4i32:
+; SSSE3-FAST: # %bb.0:
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; SSSE3-FAST-NEXT: paddd %xmm4, %xmm0
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; SSSE3-FAST-NEXT: paddd %xmm1, %xmm4
+; SSSE3-FAST-NEXT: phaddd %xmm4, %xmm0
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; SSSE3-FAST-NEXT: paddd %xmm2, %xmm1
+; SSSE3-FAST-NEXT: pshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
+; SSSE3-FAST-NEXT: paddd %xmm3, %xmm2
+; SSSE3-FAST-NEXT: phaddd %xmm2, %xmm1
+; SSSE3-FAST-NEXT: shufps {{.*#+}} xmm0 = xmm0[0,2],xmm1[0,2]
+; SSSE3-FAST-NEXT: retq
+;
+; AVX-SLOW-LABEL: reduction_sum_v4i32_v4i32:
+; AVX-SLOW: # %bb.0:
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[1,1,1,1]
+; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[1,1,1,1]
+; AVX-SLOW-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vpunpckldq {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1]
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; AVX-SLOW-NEXT: vpaddd %xmm1, %xmm2, %xmm1
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm1[1,1,1,1]
+; AVX-SLOW-NEXT: vpaddd %xmm2, %xmm1, %xmm1
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
+; AVX-SLOW-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX-SLOW-NEXT: vpshufd {{.*#+}} xmm3 = xmm2[1,1,1,1]
+; AVX-SLOW-NEXT: vpaddd %xmm3, %xmm2, %xmm2
+; AVX-SLOW-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1]
+; AVX-SLOW-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-SLOW-NEXT: retq
+;
+; AVX-FAST-LABEL: reduction_sum_v4i32_v4i32:
+; AVX-FAST: # %bb.0:
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm0[2,3,2,3]
+; AVX-FAST-NEXT: vpaddd %xmm4, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm4 = xmm1[2,3,2,3]
+; AVX-FAST-NEXT: vpaddd %xmm4, %xmm1, %xmm1
+; AVX-FAST-NEXT: vphaddd %xmm1, %xmm0, %xmm0
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,2,1,3]
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm2[2,3,2,3]
+; AVX-FAST-NEXT: vpaddd %xmm1, %xmm2, %xmm1
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm2 = xmm3[2,3,2,3]
+; AVX-FAST-NEXT: vpaddd %xmm2, %xmm3, %xmm2
+; AVX-FAST-NEXT: vphaddd %xmm2, %xmm1, %xmm1
+; AVX-FAST-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,2,1,3]
+; AVX-FAST-NEXT: vpunpcklqdq {{.*#+}} xmm0 = xmm0[0],xmm1[0]
+; AVX-FAST-NEXT: retq
+ %5 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %0)
+ %6 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %1)
+ %7 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %2)
+ %8 = call i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32> %3)
+ %9 = insertelement <4 x i32> undef, i32 %5, i32 0
+ %10 = insertelement <4 x i32> %9, i32 %6, i32 1
+ %11 = insertelement <4 x i32> %10, i32 %7, i32 2
+ %12 = insertelement <4 x i32> %11, i32 %8, i32 3
+ ret <4 x i32> %12
+}
+declare i32 @llvm.vector.reduce.add.i32.v4i32(<4 x i32>)
More information about the llvm-commits
mailing list