[llvm] [X86][PartialReduction] Lower zext-byte add reductions to vpsadbw (PR #201076)
Simon Pilgrim via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 14 00:46:16 PDT 2026
https://github.com/RKSimon updated https://github.com/llvm/llvm-project/pull/201076
>From d1f6a75c9aa52674c4127febc6b38b145a00e78e Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Sun, 7 Jun 2026 13:06:26 +0530
Subject: [PATCH 1/4] [X86] Add CodeGen tests for byte-sum reduction patterns
(NFC)
Add byte-sum.ll exercising zext-byte add-reduction shapes that could
benefit from PSADBW lowering, plus negative cases (sext, i16 source,
i16 accumulator, non-zext leaf, VF < 16).
Tests use @llvm.vector.reduce.add.* intrinsics and cover SSE2, AVX2,
and AVX-512BW subtargets. CHECK lines reflect current trunk codegen
(vpmovzxbd + vpaddd widening); a follow-up commit will add the
matcher and update the CHECK lines.
---
llvm/test/CodeGen/X86/byte-sum.ll | 1181 +++++++++++++++++++++++++++++
1 file changed, 1181 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/byte-sum.ll
diff --git a/llvm/test/CodeGen/X86/byte-sum.ll b/llvm/test/CodeGen/X86/byte-sum.ll
new file mode 100644
index 0000000000000..f37ebb0a4cb3f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/byte-sum.ll
@@ -0,0 +1,1181 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX512BW
+
+; CodeGen tests for X86PartialReduction::tryByteSumReplacement.
+; Positive cases should lower to vpsadbw; negative cases should not.
+
+ at a = global [1024 x i8] zeroinitializer, align 16
+
+;============================================================================
+; POSITIVE: zext <16 x i8> to <16 x i32>, i32 accumulator
+;============================================================================
+define i32 @byte_sum_v16_i32() nounwind {
+; SSE2-LABEL: byte_sum_v16_i32:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB0_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm6, %xmm7
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; SSE2-NEXT: paddd %xmm7, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; SSE2-NEXT: paddd %xmm6, %xmm0
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; SSE2-NEXT: paddd %xmm6, %xmm4
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; SSE2-NEXT: paddd %xmm5, %xmm3
+; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: jne .LBB0_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm4, %xmm1
+; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v16_i32:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB0_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: jne .LBB0_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v16_i32:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB0_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: jne .LBB0_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <16 x i8>, ptr %p, align 16
+ %z = zext <16 x i8> %wide.load to <16 x i32>
+ %add = add nsw <16 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
+ ret i32 %ext
+}
+
+;============================================================================
+; POSITIVE: zext <32 x i8> to <32 x i32>, i32 accumulator
+;============================================================================
+define i32 @byte_sum_v32_i32() nounwind {
+; SSE2-LABEL: byte_sum_v32_i32:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm7, %xmm7
+; SSE2-NEXT: pxor %xmm6, %xmm6
+; SSE2-NEXT: pxor %xmm8, %xmm8
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB1_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm9
+; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm10
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm4[8],xmm11[9],xmm4[9],xmm11[10],xmm4[10],xmm11[11],xmm4[11],xmm11[12],xmm4[12],xmm11[13],xmm4[13],xmm11[14],xmm4[14],xmm11[15],xmm4[15]
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm4[4],xmm12[5],xmm4[5],xmm12[6],xmm4[6],xmm12[7],xmm4[7]
+; SSE2-NEXT: paddd %xmm12, %xmm8
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
+; SSE2-NEXT: paddd %xmm11, %xmm6
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]
+; SSE2-NEXT: paddd %xmm11, %xmm7
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3]
+; SSE2-NEXT: paddd %xmm10, %xmm5
+; SSE2-NEXT: movdqa %xmm9, %xmm10
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
+; SSE2-NEXT: paddd %xmm11, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
+; SSE2-NEXT: paddd %xmm10, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15]
+; SSE2-NEXT: movdqa %xmm9, %xmm10
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3]
+; SSE2-NEXT: paddd %xmm10, %xmm0
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm4[4],xmm9[5],xmm4[5],xmm9[6],xmm4[6],xmm9[7],xmm4[7]
+; SSE2-NEXT: paddd %xmm9, %xmm2
+; SSE2-NEXT: addq $32, %rax
+; SSE2-NEXT: jne .LBB1_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm7, %xmm3
+; SSE2-NEXT: paddd %xmm8, %xmm2
+; SSE2-NEXT: paddd %xmm3, %xmm2
+; SSE2-NEXT: paddd %xmm5, %xmm1
+; SSE2-NEXT: paddd %xmm6, %xmm0
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v32_i32:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB1_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm2, %ymm4, %ymm2
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm1, %ymm4, %ymm1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm0, %ymm4, %ymm0
+; AVX2-NEXT: addq $32, %rax
+; AVX2-NEXT: jne .LBB1_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpaddd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v32_i32:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB1_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT: addq $32, %rax
+; AVX512BW-NEXT: jne .LBB1_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <32 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <32 x i8>, ptr %p, align 16
+ %z = zext <32 x i8> %wide.load to <32 x i32>
+ %add = add nsw <32 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 32
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %add)
+ ret i32 %ext
+}
+
+;============================================================================
+; POSITIVE: zext <64 x i8> to <64 x i32>, i32 accumulator
+;============================================================================
+define i32 @byte_sum_v64_i32() nounwind {
+; SSE2-LABEL: byte_sum_v64_i32:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm10, %xmm10
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm9, %xmm9
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm13, %xmm13
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: pxor %xmm12, %xmm12
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm15, %xmm15
+; SSE2-NEXT: pxor %xmm7, %xmm7
+; SSE2-NEXT: pxor %xmm6, %xmm6
+; SSE2-NEXT: pxor %xmm11, %xmm11
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm8, %xmm8
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: pxor %xmm14, %xmm14
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB2_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa 1072(%rcx,%rax), %xmm3
+; SSE2-NEXT: movdqa %xmm3, %xmm0
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm10[8],xmm0[9],xmm10[9],xmm0[10],xmm10[10],xmm0[11],xmm10[11],xmm0[12],xmm10[12],xmm0[13],xmm10[13],xmm0[14],xmm10[14],xmm0[15],xmm10[15]
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
+; SSE2-NEXT: paddd %xmm1, %xmm14
+; SSE2-NEXT: movdqa 1056(%rcx,%rax), %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; SSE2-NEXT: paddd %xmm0, %xmm4
+; SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3],xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
+; SSE2-NEXT: movdqa %xmm3, %xmm0
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT: paddd %xmm0, %xmm8
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm3, %xmm2
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm10[8],xmm3[9],xmm10[9],xmm3[10],xmm10[10],xmm3[11],xmm10[11],xmm3[12],xmm10[12],xmm3[13],xmm10[13],xmm3[14],xmm10[14],xmm3[15],xmm10[15]
+; SSE2-NEXT: movdqa %xmm3, %xmm0
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT: paddd %xmm0, %xmm11
+; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm0
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm3, %xmm6
+; SSE2-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3],xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
+; SSE2-NEXT: paddd %xmm3, %xmm7
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm1, %xmm15
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm10[8],xmm3[9],xmm10[9],xmm3[10],xmm10[10],xmm3[11],xmm10[11],xmm3[12],xmm10[12],xmm3[13],xmm10[13],xmm3[14],xmm10[14],xmm3[15],xmm10[15]
+; SSE2-NEXT: movdqa %xmm3, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
+; SSE2-NEXT: paddd %xmm1, %xmm5
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm3, %xmm12
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
+; SSE2-NEXT: movdqa %xmm2, %xmm6
+; SSE2-NEXT: movdqa %xmm15, %xmm2
+; SSE2-NEXT: movdqa %xmm14, %xmm15
+; SSE2-NEXT: movdqa %xmm13, %xmm14
+; SSE2-NEXT: movdqa %xmm11, %xmm13
+; SSE2-NEXT: movdqa %xmm12, %xmm11
+; SSE2-NEXT: movdqa %xmm8, %xmm12
+; SSE2-NEXT: movdqa %xmm9, %xmm8
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; SSE2-NEXT: paddd %xmm3, %xmm4
+; SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; SSE2-NEXT: movdqa %xmm8, %xmm9
+; SSE2-NEXT: movdqa %xmm12, %xmm8
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: movdqa %xmm13, %xmm11
+; SSE2-NEXT: movdqa %xmm14, %xmm13
+; SSE2-NEXT: movdqa %xmm15, %xmm14
+; SSE2-NEXT: movdqa %xmm2, %xmm15
+; SSE2-NEXT: movdqa %xmm6, %xmm2
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm0, %xmm3
+; SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm3, %xmm9
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT: paddd %xmm0, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm10[8],xmm1[9],xmm10[9],xmm1[10],xmm10[10],xmm1[11],xmm10[11],xmm1[12],xmm10[12],xmm1[13],xmm10[13],xmm1[14],xmm10[14],xmm1[15],xmm10[15]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm0, %xmm13
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT: addq $64, %rax
+; SSE2-NEXT: jne .LBB2_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm6, %xmm13
+; SSE2-NEXT: paddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
+; SSE2-NEXT: paddd %xmm13, %xmm12
+; SSE2-NEXT: paddd %xmm15, %xmm9
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: paddd %xmm9, %xmm0
+; SSE2-NEXT: paddd %xmm12, %xmm0
+; SSE2-NEXT: paddd %xmm11, %xmm1
+; SSE2-NEXT: paddd %xmm14, %xmm5
+; SSE2-NEXT: paddd %xmm1, %xmm5
+; SSE2-NEXT: paddd %xmm7, %xmm3
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT: paddd %xmm8, %xmm1
+; SSE2-NEXT: paddd %xmm3, %xmm1
+; SSE2-NEXT: paddd %xmm5, %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v64_i32:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpxor %xmm6, %xmm6, %xmm6
+; AVX2-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX2-NEXT: vpxor %xmm7, %xmm7, %xmm7
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB2_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm7, %ymm8, %ymm7
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm5, %ymm8, %ymm5
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm6, %ymm8, %ymm6
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm3, %ymm8, %ymm3
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm4, %ymm8, %ymm4
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm2, %ymm8, %ymm2
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm1, %ymm8, %ymm1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm0, %ymm8, %ymm0
+; AVX2-NEXT: addq $64, %rax
+; AVX2-NEXT: jne .LBB2_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddd %ymm6, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm7, %ymm4, %ymm4
+; AVX2-NEXT: vpaddd %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpaddd %ymm5, %ymm2, %ymm2
+; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v64_i32:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB2_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm3, %zmm4, %zmm3
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm2, %zmm4, %zmm2
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm4, %zmm1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm4, %zmm0
+; AVX512BW-NEXT: addq $64, %rax
+; AVX512BW-NEXT: jne .LBB2_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vpaddd %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT: vpaddd %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <64 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <64 x i8>, ptr %p, align 16
+ %z = zext <64 x i8> %wide.load to <64 x i32>
+ %add = add nsw <64 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 64
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %add)
+ ret i32 %ext
+}
+
+;============================================================================
+; POSITIVE: zext <16 x i8> to <16 x i64>, i64 accumulator
+;============================================================================
+define i64 @byte_sum_v16_i64() nounwind {
+; SSE2-LABEL: byte_sum_v16_i64:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm7, %xmm7
+; SSE2-NEXT: pxor %xmm6, %xmm6
+; SSE2-NEXT: pxor %xmm8, %xmm8
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB3_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm10
+; SSE2-NEXT: movdqa %xmm10, %xmm9
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15]
+; SSE2-NEXT: movdqa %xmm9, %xmm11
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm12 = xmm12[2],xmm4[2],xmm12[3],xmm4[3]
+; SSE2-NEXT: paddq %xmm12, %xmm8
+; SSE2-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1]
+; SSE2-NEXT: paddq %xmm11, %xmm6
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; SSE2-NEXT: movdqa %xmm9, %xmm11
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm4[2],xmm11[3],xmm4[3]
+; SSE2-NEXT: paddq %xmm11, %xmm7
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm4[0],xmm12[1],xmm4[1]
+; SSE2-NEXT: paddq %xmm12, %xmm1
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm4[2],xmm11[3],xmm4[3]
+; SSE2-NEXT: paddq %xmm11, %xmm3
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1]
+; SSE2-NEXT: paddq %xmm11, %xmm0
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm4[2],xmm10[3],xmm4[3]
+; SSE2-NEXT: paddq %xmm10, %xmm2
+; SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
+; SSE2-NEXT: paddq %xmm9, %xmm5
+; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: jne .LBB3_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddq %xmm7, %xmm3
+; SSE2-NEXT: paddq %xmm8, %xmm2
+; SSE2-NEXT: paddq %xmm3, %xmm2
+; SSE2-NEXT: paddq %xmm5, %xmm1
+; SSE2-NEXT: paddq %xmm6, %xmm0
+; SSE2-NEXT: paddq %xmm1, %xmm0
+; SSE2-NEXT: paddq %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddq %xmm0, %xmm1
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v16_i64:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB3_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpaddq %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpaddq %ymm2, %ymm4, %ymm2
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpaddq %ymm1, %ymm4, %ymm1
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpaddq %ymm0, %ymm4, %ymm0
+; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: jne .LBB3_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddq %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpaddq %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v16_i64:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB3_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxbq {{.*#+}} zmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT: vpaddq %zmm1, %zmm2, %zmm1
+; AVX512BW-NEXT: vpmovzxbq {{.*#+}} zmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT: vpaddq %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: jne .LBB3_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vpaddq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovq %xmm0, %rax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <16 x i8>, ptr %p, align 16
+ %z = zext <16 x i8> %wide.load to <16 x i64>
+ %add = add nsw <16 x i64> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %add)
+ ret i64 %ext
+}
+
+;============================================================================
+; NEGATIVE: sext instead of zext
+;============================================================================
+define i32 @byte_sum_v16_i32_sext() nounwind {
+; SSE2-LABEL: byte_sum_v16_i32_sext:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB4_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; SSE2-NEXT: psrad $24, %xmm6
+; SSE2-NEXT: paddd %xmm6, %xmm0
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
+; SSE2-NEXT: psrad $24, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm1
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; SSE2-NEXT: psrad $24, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm3
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; SSE2-NEXT: psrad $24, %xmm4
+; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: jne .LBB4_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm2, %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v16_i32_sext:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB4_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovsxbd 1024(%rcx,%rax), %ymm2
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpmovsxbd 1032(%rcx,%rax), %ymm2
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: jne .LBB4_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v16_i32_sext:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB4_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovsxbd 1024(%rcx,%rax), %zmm1
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: jne .LBB4_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <16 x i8>, ptr %p, align 16
+ %z = sext <16 x i8> %wide.load to <16 x i32>
+ %add = add nsw <16 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
+ ret i32 %ext
+}
+
+;============================================================================
+; NEGATIVE: i16 source instead of i8
+;============================================================================
+define i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
+; SSE2-LABEL: word_sum_v16_i32:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movq $-2048, %rax # imm = 0xF800
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB5_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqu 2048(%rdi,%rax), %xmm5
+; SSE2-NEXT: movdqu 2064(%rdi,%rax), %xmm6
+; SSE2-NEXT: movdqa %xmm6, %xmm7
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
+; SSE2-NEXT: paddd %xmm7, %xmm3
+; SSE2-NEXT: movdqa %xmm5, %xmm7
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; SSE2-NEXT: paddd %xmm7, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; SSE2-NEXT: paddd %xmm5, %xmm0
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; SSE2-NEXT: paddd %xmm6, %xmm4
+; SSE2-NEXT: addq $32, %rax
+; SSE2-NEXT: jne .LBB5_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm4, %xmm1
+; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: word_sum_v16_i32:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-2048, %rax # imm = 0xF800
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB5_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: addq $32, %rax
+; AVX2-NEXT: jne .LBB5_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: word_sum_v16_i32:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-2048, %rax # imm = 0xF800
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB5_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: addq $32, %rax
+; AVX512BW-NEXT: jne .LBB5_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds i16, ptr %src, i64 %index
+ %wide.load = load <16 x i16>, ptr %p, align 2
+ %z = zext <16 x i16> %wide.load to <16 x i32>
+ %add = add nsw <16 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
+ ret i32 %ext
+}
+
+;============================================================================
+; NEGATIVE: i16 accumulator (element type < 32 bits)
+;============================================================================
+define i16 @byte_sum_v16_i16() nounwind {
+; SSE2-LABEL: byte_sum_v16_i16:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB6_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm3
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
+; SSE2-NEXT: paddw %xmm4, %xmm0
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; SSE2-NEXT: paddw %xmm3, %xmm2
+; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: jne .LBB6_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddw %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddw %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddw %xmm1, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: psrld $16, %xmm1
+; SSE2-NEXT: paddw %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v16_i16:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB6_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; AVX2-NEXT: vpaddw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: jne .LBB6_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v16_i16:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB6_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; AVX512BW-NEXT: vpaddw %ymm0, %ymm1, %ymm0
+; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: jne .LBB6_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX512BW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i16> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <16 x i8>, ptr %p, align 16
+ %z = zext <16 x i8> %wide.load to <16 x i16>
+ %add = add nsw <16 x i16> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %add)
+ ret i16 %ext
+}
+
+;============================================================================
+; NEGATIVE: leaf is not a zext (already i32 values)
+;============================================================================
+define i32 @dword_sum_v16_i32(ptr nocapture readonly %src) nounwind {
+; SSE2-LABEL: dword_sum_v16_i32:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movq $-4096, %rax # imm = 0xF000
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB7_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqu 4096(%rdi,%rax), %xmm4
+; SSE2-NEXT: paddd %xmm4, %xmm0
+; SSE2-NEXT: movdqu 4112(%rdi,%rax), %xmm4
+; SSE2-NEXT: paddd %xmm4, %xmm1
+; SSE2-NEXT: movdqu 4128(%rdi,%rax), %xmm4
+; SSE2-NEXT: paddd %xmm4, %xmm3
+; SSE2-NEXT: movdqu 4144(%rdi,%rax), %xmm4
+; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: addq $64, %rax
+; SSE2-NEXT: jne .LBB7_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm2, %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: dword_sum_v16_i32:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-4096, %rax # imm = 0xF000
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB7_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpaddd 4096(%rdi,%rax), %ymm1, %ymm1
+; AVX2-NEXT: vpaddd 4128(%rdi,%rax), %ymm0, %ymm0
+; AVX2-NEXT: addq $64, %rax
+; AVX2-NEXT: jne .LBB7_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: dword_sum_v16_i32:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-4096, %rax # imm = 0xF000
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB7_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpaddd 4096(%rdi,%rax), %zmm0, %zmm0
+; AVX512BW-NEXT: addq $64, %rax
+; AVX512BW-NEXT: jne .LBB7_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds i32, ptr %src, i64 %index
+ %wide.load = load <16 x i32>, ptr %p, align 4
+ %add = add nsw <16 x i32> %wide.load, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
+ ret i32 %ext
+}
+
+;============================================================================
+; NEGATIVE: VF = 8 (fewer than 16 lanes)
+;============================================================================
+define i32 @byte_sum_v8_i32() nounwind {
+; SSE2-LABEL: byte_sum_v8_i32:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB8_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movq {{.*#+}} xmm3 = mem[0],zero
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSE2-NEXT: paddd %xmm4, %xmm0
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT: paddd %xmm3, %xmm2
+; SSE2-NEXT: addq $8, %rax
+; SSE2-NEXT: jne .LBB8_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v8_i32:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB8_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: addq $8, %rax
+; AVX2-NEXT: jne .LBB8_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v8_i32:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB8_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512BW-NEXT: addq $8, %rax
+; AVX512BW-NEXT: jne .LBB8_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <8 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <8 x i8>, ptr %p, align 8
+ %z = zext <8 x i8> %wide.load to <8 x i32>
+ %add = add nsw <8 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 8
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %add)
+ ret i32 %ext
+}
>From 2895c930fa867542a6d75b4dcc357a84d21af5e4 Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Sun, 7 Jun 2026 13:20:57 +0530
Subject: [PATCH 2/4] [X86][PartialReduction] Lower zext-byte add reductions to
vpsadbw
Loops of the form `for (i) sum += bytes[i];` (uint8_t input, i32 or
i64 accumulator) currently lower to vpmovzxbd + vpaddd even though
PSADBW(x, 0) computes the same sum natively in one instruction per
128/256/512-bit lane.
Teach X86PartialReduction's matcher to rewrite the
`zext <N x i8> to <N x i32|i64>` leaves of an add reduction
(N >= 16) into PSADBW(x, 0). The rewrite splits the source into as
many SSE2/AVX2/AVX-512BW lanes as the subtarget and VF require;
i64 accumulators consume PSADBW's natural <N/8 x i64> output
directly without an intermediate i32 bitcast.
x86-partial-reduction-byte-sum*.ll isolate the matcher with
`opt -passes=x86-partial-reduction` (positive / negative / wide-i64
/ debug-location preservation). byte-sum.ll exercises the full
CodeGen pipeline with `llc` on +sse2, +avx2 and +avx512bw.
---
llvm/lib/Target/X86/X86PartialReduction.cpp | 116 ++++-
llvm/test/CodeGen/X86/byte-sum.ll | 400 ++++--------------
...x86-partial-reduction-byte-sum-debugloc.ll | 43 ++
...x86-partial-reduction-byte-sum-negative.ll | 49 +++
...x86-partial-reduction-byte-sum-wide-i64.ll | 59 +++
.../X86/x86-partial-reduction-byte-sum.ll | 26 ++
6 files changed, 386 insertions(+), 307 deletions(-)
create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll
create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll
create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll
create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
diff --git a/llvm/lib/Target/X86/X86PartialReduction.cpp b/llvm/lib/Target/X86/X86PartialReduction.cpp
index a017ed9f2ff50..85092ae569afd 100644
--- a/llvm/lib/Target/X86/X86PartialReduction.cpp
+++ b/llvm/lib/Target/X86/X86PartialReduction.cpp
@@ -18,6 +18,7 @@
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/Analysis.h"
#include "llvm/IR/Constants.h"
+#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsX86.h"
@@ -44,6 +45,7 @@ class X86PartialReduction {
private:
bool tryMAddReplacement(Instruction *Op, bool ReduceInOneBB);
bool trySADReplacement(Instruction *Op);
+ bool tryByteSumReplacement(Instruction *Op);
};
class X86PartialReductionLegacy : public FunctionPass {
@@ -353,6 +355,106 @@ bool X86PartialReduction::trySADReplacement(Instruction *Op) {
return true;
}
+bool X86PartialReduction::tryByteSumReplacement(Instruction *Op) {
+ if (!ST->hasSSE2())
+ return false;
+
+ auto *OpTy = dyn_cast<FixedVectorType>(Op->getType());
+ if (!OpTy)
+ return false;
+ unsigned ElemBits = OpTy->getElementType()->getScalarSizeInBits();
+ if (ElemBits != 32 && ElemBits != 64)
+ return false;
+
+ auto *ZExt = dyn_cast<ZExtInst>(Op);
+ if (!ZExt)
+ return false;
+
+ auto *SrcTy = dyn_cast<FixedVectorType>(ZExt->getOperand(0)->getType());
+ if (!SrcTy || !SrcTy->getElementType()->isIntegerTy(8))
+ return false;
+
+ unsigned NumElts = OpTy->getNumElements();
+
+ // Below 16 elements, SelectionDAG's SAD matcher handles it.
+ if (NumElts < 16)
+ return false;
+
+ // Select the widest psadbw intrinsic the subtarget supports.
+ unsigned IntrinsicNumElts;
+ Intrinsic::ID IID;
+ if (ST->useBWIRegs() && NumElts >= 64) {
+ IID = Intrinsic::x86_avx512_psad_bw_512;
+ IntrinsicNumElts = 64;
+ } else if (ST->hasAVX2() && NumElts >= 32) {
+ IID = Intrinsic::x86_avx2_psad_bw;
+ IntrinsicNumElts = 32;
+ } else {
+ IID = Intrinsic::x86_sse2_psad_bw;
+ IntrinsicNumElts = 16;
+ }
+
+ if (NumElts % IntrinsicNumElts != 0 ||
+ !isPowerOf2_32(NumElts / IntrinsicNumElts))
+ return false;
+ unsigned NumSplits = NumElts / IntrinsicNumElts;
+
+ IRBuilder<> Builder(Op);
+ Builder.SetCurrentDebugLocation(Op->getDebugLoc());
+
+ Function *PSADBWFn = Intrinsic::getOrInsertDeclaration(Op->getModule(), IID);
+
+ // psadbw(x, 0) horizontally sums 8 bytes per lane into i64.
+ auto *I8VecTy = FixedVectorType::get(Builder.getInt8Ty(), IntrinsicNumElts);
+ Value *Zeroes = Constant::getNullValue(I8VecTy);
+
+ // For i32 accumulators, bitcast each i64 lane to two i32 lanes.
+ // Per-lane sums are at most 8*255 = 2040, so the upper i32 is always zero.
+ FixedVectorType *I32PerSplitTy =
+ ElemBits == 32
+ ? FixedVectorType::get(Builder.getInt32Ty(), IntrinsicNumElts / 4)
+ : nullptr;
+
+ // Split input into IntrinsicNumElts-byte lanes and compute psadbw per lane.
+ Value *Src = ZExt->getOperand(0);
+ SmallVector<Value *, 4> Ops(NumSplits);
+ for (unsigned i = 0; i != NumSplits; ++i) {
+ SmallVector<int, 64> ExtractMask(IntrinsicNumElts);
+ std::iota(ExtractMask.begin(), ExtractMask.end(), i * IntrinsicNumElts);
+ Value *ExtractSrc = Builder.CreateShuffleVector(Src, Src, ExtractMask);
+ Ops[i] = Builder.CreateCall(PSADBWFn, {ExtractSrc, Zeroes});
+ if (I32PerSplitTy)
+ Ops[i] = Builder.CreateBitCast(Ops[i], I32PerSplitTy);
+ }
+
+ // Concat per-split results with a pairwise shuffle tree.
+ unsigned Stages = Log2_32(NumSplits);
+ for (unsigned S = Stages; S > 0; --S) {
+ unsigned NumConcatElts =
+ cast<FixedVectorType>(Ops[0]->getType())->getNumElements() * 2;
+ for (unsigned i = 0; i != 1U << (S - 1); ++i) {
+ SmallVector<int, 64> ConcatMask(NumConcatElts);
+ std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
+ Ops[i] =
+ Builder.CreateShuffleVector(Ops[i * 2], Ops[i * 2 + 1], ConcatMask);
+ }
+ }
+
+ // Pad with zeros to match the original vector width.
+ SmallVector<int, 32> ConcatMask(NumElts);
+ unsigned SubElts = cast<FixedVectorType>(Ops[0]->getType())->getNumElements();
+ for (unsigned i = 0; i != SubElts; ++i)
+ ConcatMask[i] = i;
+ for (unsigned i = SubElts; i != NumElts; ++i)
+ ConcatMask[i] = (i % SubElts) + SubElts;
+ Value *Zero = Constant::getNullValue(Ops[0]->getType());
+ Ops[0] = Builder.CreateShuffleVector(Ops[0], Zero, ConcatMask);
+
+ Op->replaceAllUsesWith(Ops[0]);
+ Op->eraseFromParent();
+ return true;
+}
+
// Walk backwards from the ExtractElementInst and determine if it is the end of
// a horizontal reduction. Return the input to the reduction if we find one.
static Value *matchAddReduction(const ExtractElementInst &EE,
@@ -530,8 +632,20 @@ bool X86PartialReduction::run(Function &F) {
// Don't do SAD matching on the root node. SelectionDAG already
// has support for that and currently generates better code.
- if (I != Root && trySADReplacement(I))
+ if (I != Root && trySADReplacement(I)) {
MadeChange = true;
+ continue;
+ }
+
+ // Byte sum via psadbw(x, 0). Same rationale as trySADReplacement:
+ // don't match on the root node because SelectionDAG already handles
+ // small single-vector patterns and generally emits better code for
+ // them. We only help on wider intermediate shapes that reach us
+ // from loop vectorization.
+ if (I != Root && tryByteSumReplacement(I)) {
+ MadeChange = true;
+ continue;
+ }
}
}
}
diff --git a/llvm/test/CodeGen/X86/byte-sum.ll b/llvm/test/CodeGen/X86/byte-sum.ll
index f37ebb0a4cb3f..0a22fe31b46f7 100644
--- a/llvm/test/CodeGen/X86/byte-sum.ll
+++ b/llvm/test/CodeGen/X86/byte-sum.ll
@@ -14,35 +14,21 @@
define i32 @byte_sum_v16_i32() nounwind {
; SSE2-LABEL: byte_sum_v16_i32:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: pxor %xmm4, %xmm4
-; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB0_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; SSE2-NEXT: movdqa %xmm6, %xmm7
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
-; SSE2-NEXT: paddd %xmm7, %xmm1
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; SSE2-NEXT: paddd %xmm6, %xmm0
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; SSE2-NEXT: paddd %xmm6, %xmm4
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
-; SSE2-NEXT: paddd %xmm5, %xmm3
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm2
+; SSE2-NEXT: psadbw %xmm0, %xmm2
+; SSE2-NEXT: paddd %xmm2, %xmm1
; SSE2-NEXT: addq $16, %rax
; SSE2-NEXT: jne .LBB0_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm4, %xmm1
-; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm0
; SSE2-NEXT: paddd %xmm1, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddd %xmm0, %xmm1
@@ -57,17 +43,16 @@ define i32 @byte_sum_v16_i32() nounwind {
; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB0_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm3
+; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; AVX2-NEXT: addq $16, %rax
; AVX2-NEXT: jne .LBB0_1
; AVX2-NEXT: # %bb.2: # %middle.block
-; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -83,11 +68,12 @@ define i32 @byte_sum_v16_i32() nounwind {
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB0_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm2
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
; AVX512BW-NEXT: addq $16, %rax
; AVX512BW-NEXT: jne .LBB0_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
@@ -127,58 +113,29 @@ middle.block:
define i32 @byte_sum_v32_i32() nounwind {
; SSE2-LABEL: byte_sum_v32_i32:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: pxor %xmm7, %xmm7
-; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pxor %xmm8, %xmm8
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB1_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm9
-; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm10
-; SSE2-NEXT: movdqa %xmm10, %xmm11
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm4[8],xmm11[9],xmm4[9],xmm11[10],xmm4[10],xmm11[11],xmm4[11],xmm11[12],xmm4[12],xmm11[13],xmm4[13],xmm11[14],xmm4[14],xmm11[15],xmm4[15]
-; SSE2-NEXT: movdqa %xmm11, %xmm12
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm4[4],xmm12[5],xmm4[5],xmm12[6],xmm4[6],xmm12[7],xmm4[7]
-; SSE2-NEXT: paddd %xmm12, %xmm8
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
-; SSE2-NEXT: paddd %xmm11, %xmm6
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; SSE2-NEXT: movdqa %xmm10, %xmm11
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]
-; SSE2-NEXT: paddd %xmm11, %xmm7
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3]
-; SSE2-NEXT: paddd %xmm10, %xmm5
-; SSE2-NEXT: movdqa %xmm9, %xmm10
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; SSE2-NEXT: movdqa %xmm10, %xmm11
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
-; SSE2-NEXT: paddd %xmm11, %xmm1
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; SSE2-NEXT: paddd %xmm10, %xmm3
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15]
-; SSE2-NEXT: movdqa %xmm9, %xmm10
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3]
-; SSE2-NEXT: paddd %xmm10, %xmm0
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm4[4],xmm9[5],xmm4[5],xmm9[6],xmm4[6],xmm9[7],xmm4[7]
-; SSE2-NEXT: paddd %xmm9, %xmm2
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm3
+; SSE2-NEXT: psadbw %xmm0, %xmm3
+; SSE2-NEXT: paddd %xmm3, %xmm1
+; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm3
+; SSE2-NEXT: psadbw %xmm0, %xmm3
+; SSE2-NEXT: paddd %xmm3, %xmm2
; SSE2-NEXT: addq $32, %rax
; SSE2-NEXT: jne .LBB1_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm7, %xmm3
-; SSE2-NEXT: paddd %xmm8, %xmm2
-; SSE2-NEXT: paddd %xmm3, %xmm2
-; SSE2-NEXT: paddd %xmm5, %xmm1
-; SSE2-NEXT: paddd %xmm6, %xmm0
-; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: paddd %xmm0, %xmm2
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm0
+; SSE2-NEXT: paddd %xmm0, %xmm1
; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: paddd %xmm1, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddd %xmm0, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
@@ -192,24 +149,16 @@ define i32 @byte_sum_v32_i32() nounwind {
; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB1_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm2, %ymm4, %ymm2
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm1, %ymm4, %ymm1
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm0, %ymm4, %ymm0
+; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %ymm0, %ymm2
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
; AVX2-NEXT: addq $32, %rax
; AVX2-NEXT: jne .LBB1_1
; AVX2-NEXT: # %bb.2: # %middle.block
-; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpaddd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm1
+; AVX2-NEXT: vpaddd %ymm0, %ymm0, %ymm0
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
@@ -227,17 +176,16 @@ define i32 @byte_sum_v32_i32() nounwind {
; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB1_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %ymm1, %ymm3
+; AVX512BW-NEXT: vpaddd %zmm2, %zmm3, %zmm2
; AVX512BW-NEXT: addq $32, %rax
; AVX512BW-NEXT: jne .LBB1_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
-; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
@@ -274,137 +222,46 @@ middle.block:
define i32 @byte_sum_v64_i32() nounwind {
; SSE2-LABEL: byte_sum_v64_i32:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm10, %xmm10
+; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
-; SSE2-NEXT: pxor %xmm9, %xmm9
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pxor %xmm13, %xmm13
-; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: pxor %xmm12, %xmm12
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: pxor %xmm15, %xmm15
-; SSE2-NEXT: pxor %xmm7, %xmm7
-; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pxor %xmm11, %xmm11
+; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm8, %xmm8
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: pxor %xmm14, %xmm14
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB2_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa 1072(%rcx,%rax), %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm0
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm10[8],xmm0[9],xmm10[9],xmm0[10],xmm10[10],xmm0[11],xmm10[11],xmm0[12],xmm10[12],xmm0[13],xmm10[13],xmm0[14],xmm10[14],xmm0[15],xmm10[15]
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm1, %xmm14
-; SSE2-NEXT: movdqa 1056(%rcx,%rax), %xmm1
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; SSE2-NEXT: paddd %xmm0, %xmm4
-; SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3],xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
-; SSE2-NEXT: movdqa %xmm3, %xmm0
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm0, %xmm8
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm1, %xmm3
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm10[8],xmm3[9],xmm10[9],xmm3[10],xmm10[10],xmm3[11],xmm10[11],xmm3[12],xmm10[12],xmm3[13],xmm10[13],xmm3[14],xmm10[14],xmm3[15],xmm10[15]
-; SSE2-NEXT: movdqa %xmm3, %xmm0
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm0, %xmm11
-; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm3, %xmm6
-; SSE2-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3],xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
-; SSE2-NEXT: movdqa %xmm1, %xmm3
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm3, %xmm7
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm1, %xmm15
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm10[8],xmm3[9],xmm10[9],xmm3[10],xmm10[10],xmm3[11],xmm10[11],xmm3[12],xmm10[12],xmm3[13],xmm10[13],xmm3[14],xmm10[14],xmm3[15],xmm10[15]
-; SSE2-NEXT: movdqa %xmm3, %xmm1
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm1, %xmm5
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm1
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm3, %xmm12
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
-; SSE2-NEXT: movdqa %xmm2, %xmm6
-; SSE2-NEXT: movdqa %xmm15, %xmm2
-; SSE2-NEXT: movdqa %xmm14, %xmm15
-; SSE2-NEXT: movdqa %xmm13, %xmm14
-; SSE2-NEXT: movdqa %xmm11, %xmm13
-; SSE2-NEXT: movdqa %xmm12, %xmm11
-; SSE2-NEXT: movdqa %xmm8, %xmm12
-; SSE2-NEXT: movdqa %xmm9, %xmm8
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; SSE2-NEXT: paddd %xmm3, %xmm4
-; SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; SSE2-NEXT: movdqa %xmm8, %xmm9
-; SSE2-NEXT: movdqa %xmm12, %xmm8
-; SSE2-NEXT: movdqa %xmm11, %xmm12
-; SSE2-NEXT: movdqa %xmm13, %xmm11
-; SSE2-NEXT: movdqa %xmm14, %xmm13
-; SSE2-NEXT: movdqa %xmm15, %xmm14
-; SSE2-NEXT: movdqa %xmm2, %xmm15
-; SSE2-NEXT: movdqa %xmm6, %xmm2
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm0, %xmm3
-; SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm3, %xmm9
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm0, %xmm3
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm10[8],xmm1[9],xmm10[9],xmm1[10],xmm10[10],xmm1[11],xmm10[11],xmm1[12],xmm10[12],xmm1[13],xmm10[13],xmm1[14],xmm10[14],xmm1[15],xmm10[15]
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm0, %xmm13
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: paddd %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm5
+; SSE2-NEXT: psadbw %xmm3, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm0
+; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm5
+; SSE2-NEXT: psadbw %xmm3, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm4
+; SSE2-NEXT: movdqa 1056(%rcx,%rax), %xmm5
+; SSE2-NEXT: psadbw %xmm3, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm2
+; SSE2-NEXT: movdqa 1072(%rcx,%rax), %xmm5
+; SSE2-NEXT: psadbw %xmm3, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm1
; SSE2-NEXT: addq $64, %rax
; SSE2-NEXT: jne .LBB2_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm6, %xmm13
-; SSE2-NEXT: paddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
-; SSE2-NEXT: paddd %xmm13, %xmm12
-; SSE2-NEXT: paddd %xmm15, %xmm9
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: paddd %xmm2, %xmm0
-; SSE2-NEXT: paddd %xmm9, %xmm0
-; SSE2-NEXT: paddd %xmm12, %xmm0
-; SSE2-NEXT: paddd %xmm11, %xmm1
-; SSE2-NEXT: paddd %xmm14, %xmm5
-; SSE2-NEXT: paddd %xmm1, %xmm5
-; SSE2-NEXT: paddd %xmm7, %xmm3
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; SSE2-NEXT: paddd %xmm8, %xmm1
+; SSE2-NEXT: paddd %xmm3, %xmm2
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm5
+; SSE2-NEXT: paddd %xmm3, %xmm0
; SSE2-NEXT: paddd %xmm3, %xmm1
+; SSE2-NEXT: paddd %xmm3, %xmm4
+; SSE2-NEXT: paddd %xmm5, %xmm4
; SSE2-NEXT: paddd %xmm5, %xmm1
-; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: paddd %xmm4, %xmm1
+; SSE2-NEXT: paddd %xmm5, %xmm0
+; SSE2-NEXT: paddd %xmm2, %xmm5
+; SSE2-NEXT: paddd %xmm0, %xmm5
+; SSE2-NEXT: paddd %xmm1, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; SSE2-NEXT: paddd %xmm5, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; SSE2-NEXT: paddd %xmm0, %xmm1
; SSE2-NEXT: movd %xmm1, %eax
@@ -417,40 +274,22 @@ define i32 @byte_sum_v64_i32() nounwind {
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT: vpxor %xmm6, %xmm6, %xmm6
-; AVX2-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX2-NEXT: vpxor %xmm7, %xmm7, %xmm7
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB2_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm7, %ymm8, %ymm7
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm5, %ymm8, %ymm5
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm6, %ymm8, %ymm6
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm3, %ymm8, %ymm3
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm4, %ymm8, %ymm4
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm2, %ymm8, %ymm2
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm1, %ymm8, %ymm1
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm0, %ymm8, %ymm0
+; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %ymm0, %ymm3
+; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: vpsadbw 1056(%rcx,%rax), %ymm0, %ymm3
+; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; AVX2-NEXT: addq $64, %rax
; AVX2-NEXT: jne .LBB2_1
; AVX2-NEXT: # %bb.2: # %middle.block
-; AVX2-NEXT: vpaddd %ymm6, %ymm0, %ymm0
-; AVX2-NEXT: vpaddd %ymm7, %ymm4, %ymm4
-; AVX2-NEXT: vpaddd %ymm4, %ymm0, %ymm0
-; AVX2-NEXT: vpaddd %ymm3, %ymm1, %ymm1
-; AVX2-NEXT: vpaddd %ymm5, %ymm2, %ymm2
-; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm2
+; AVX2-NEXT: vpaddd %ymm0, %ymm0, %ymm3
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vpaddd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm3, %ymm2, %ymm1
+; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -467,24 +306,16 @@ define i32 @byte_sum_v64_i32() nounwind {
; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB2_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm3, %zmm4, %zmm3
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm2, %zmm4, %zmm2
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm1, %zmm4, %zmm1
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm0, %zmm4, %zmm0
+; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %zmm0, %zmm2
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1
; AVX512BW-NEXT: addq $64, %rax
; AVX512BW-NEXT: jne .LBB2_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
-; AVX512BW-NEXT: vpaddd %zmm2, %zmm1, %zmm1
-; AVX512BW-NEXT: vpaddd %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm1
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm0, %zmm0
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
@@ -522,63 +353,28 @@ middle.block:
define i64 @byte_sum_v16_i64() nounwind {
; SSE2-LABEL: byte_sum_v16_i64:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: pxor %xmm7, %xmm7
-; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pxor %xmm8, %xmm8
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB3_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm10
-; SSE2-NEXT: movdqa %xmm10, %xmm9
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15]
-; SSE2-NEXT: movdqa %xmm9, %xmm11
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]
-; SSE2-NEXT: movdqa %xmm11, %xmm12
-; SSE2-NEXT: punpckhdq {{.*#+}} xmm12 = xmm12[2],xmm4[2],xmm12[3],xmm4[3]
-; SSE2-NEXT: paddq %xmm12, %xmm8
-; SSE2-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1]
-; SSE2-NEXT: paddq %xmm11, %xmm6
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; SSE2-NEXT: movdqa %xmm9, %xmm11
-; SSE2-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm4[2],xmm11[3],xmm4[3]
-; SSE2-NEXT: paddq %xmm11, %xmm7
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; SSE2-NEXT: movdqa %xmm10, %xmm11
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
-; SSE2-NEXT: movdqa %xmm11, %xmm12
-; SSE2-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm4[0],xmm12[1],xmm4[1]
-; SSE2-NEXT: paddq %xmm12, %xmm1
-; SSE2-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm4[2],xmm11[3],xmm4[3]
-; SSE2-NEXT: paddq %xmm11, %xmm3
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; SSE2-NEXT: movdqa %xmm10, %xmm11
-; SSE2-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1]
-; SSE2-NEXT: paddq %xmm11, %xmm0
-; SSE2-NEXT: punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm4[2],xmm10[3],xmm4[3]
-; SSE2-NEXT: paddq %xmm10, %xmm2
-; SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
-; SSE2-NEXT: paddq %xmm9, %xmm5
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm2
+; SSE2-NEXT: psadbw %xmm0, %xmm2
+; SSE2-NEXT: paddq %xmm2, %xmm1
; SSE2-NEXT: addq $16, %rax
; SSE2-NEXT: jne .LBB3_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddq %xmm7, %xmm3
-; SSE2-NEXT: paddq %xmm8, %xmm2
-; SSE2-NEXT: paddq %xmm3, %xmm2
-; SSE2-NEXT: paddq %xmm5, %xmm1
-; SSE2-NEXT: paddq %xmm6, %xmm0
-; SSE2-NEXT: paddq %xmm1, %xmm0
-; SSE2-NEXT: paddq %xmm2, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddq %xmm0, %xmm1
-; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: paddq %xmm0, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: paddq %xmm0, %xmm2
+; SSE2-NEXT: paddq %xmm0, %xmm2
+; SSE2-NEXT: paddq %xmm1, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; SSE2-NEXT: paddq %xmm2, %xmm0
+; SSE2-NEXT: movq %xmm0, %rax
; SSE2-NEXT: retq
;
; AVX2-LABEL: byte_sum_v16_i64:
@@ -588,23 +384,16 @@ define i64 @byte_sum_v16_i64() nounwind {
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB3_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpaddq %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpaddq %ymm2, %ymm4, %ymm2
-; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpaddq %ymm1, %ymm4, %ymm1
-; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpaddq %ymm0, %ymm4, %ymm0
+; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm3
+; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm2
; AVX2-NEXT: addq $16, %rax
; AVX2-NEXT: jne .LBB3_1
; AVX2-NEXT: # %bb.2: # %middle.block
-; AVX2-NEXT: vpaddq %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpaddq %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpaddq %ymm0, %ymm2, %ymm1
+; AVX2-NEXT: vpaddq %ymm0, %ymm0, %ymm0
; AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
@@ -620,17 +409,16 @@ define i64 @byte_sum_v16_i64() nounwind {
; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB3_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpmovzxbq {{.*#+}} zmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT: vpaddq %zmm1, %zmm2, %zmm1
-; AVX512BW-NEXT: vpmovzxbq {{.*#+}} zmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT: vpaddq %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm3
+; AVX512BW-NEXT: vpaddq %zmm2, %zmm3, %zmm2
; AVX512BW-NEXT: addq $16, %rax
; AVX512BW-NEXT: jne .LBB3_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
-; AVX512BW-NEXT: vpaddq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpaddq %zmm0, %zmm2, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
; AVX512BW-NEXT: vpaddq %zmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll
new file mode 100644
index 0000000000000..fcd757ba163ad
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+sse2 -S | FileCheck %s
+
+; Verify that X86PartialReduction::tryByteSumReplacement carries the
+; original add instruction's !dbg location onto the freshly emitted
+; psadbw / shuffle sequence so source locations survive into the
+; optimized IR under -g.
+
+ at a = global [1024 x i8] zeroinitializer, align 16
+
+; CHECK-LABEL: @byte_sum_v16_i32
+; CHECK: call <2 x i64> @llvm.x86.sse2.psad.bw({{.*}}), !dbg ![[#LOC:]]
+; CHECK: ![[#LOC]] = !DILocation(line: 42,
+define i32 @byte_sum_v16_i32() nounwind !dbg !6 {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <16 x i8>, ptr %p, align 16
+ %z = zext <16 x i8> %wide.load to <16 x i32>, !dbg !8
+ %add = add nsw <16 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
+ ret i32 %ext
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "byte-sum-debugloc.c", directory: "/tmp")
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = distinct !DISubprogram(name: "byte_sum_v16_i32", scope: !1, file: !1, line: 1, type: !7, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
+!7 = !DISubroutineType(types: !{})
+!8 = !DILocation(line: 42, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll
new file mode 100644
index 0000000000000..d46edfd754514
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+avx2 -S | FileCheck %s
+
+; Shapes that tryByteSumReplacement must not rewrite.
+
+ at a = global [1024 x i8] zeroinitializer, align 16
+
+; CHECK-LABEL: @byte_sum_v8_i32
+; CHECK-NOT: psad.bw
+define i32 @byte_sum_v8_i32() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <8 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <8 x i8>, ptr %p, align 8
+ %z = zext <8 x i8> %wide.load to <8 x i32>
+ %add = add nsw <8 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 8
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %add)
+ ret i32 %ext
+}
+
+; CHECK-LABEL: @byte_sum_v24_i32
+; CHECK-NOT: psad.bw
+define i32 @byte_sum_v24_i32() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <24 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <24 x i8>, ptr %p, align 8
+ %z = zext <24 x i8> %wide.load to <24 x i32>
+ %add = add nsw <24 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 24
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %add)
+ ret i32 %ext
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll
new file mode 100644
index 0000000000000..23be1178bf767
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+avx2 -S | FileCheck %s --check-prefix=AVX2
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+avx512bw -S | FileCheck %s --check-prefix=AVX512
+
+; Wider-VF i64-accumulator positive shapes for tryByteSumReplacement.
+; These exercise the PerSplitTy = <IntrinsicNumElts/8 x i64> branch where the
+; matcher must NOT bitcast the psadbw result back to i32.
+;
+; VF=32 i64 uses the AVX2 256-bit lane (one avx2.psad.bw call). On
+; +avx512bw the dispatch falls into the same AVX2 path because NumElts<64.
+;
+; VF=64 i64 uses the AVX-512BW 512-bit lane (one avx512.psad.bw.512 call).
+
+ at a = global [1024 x i8] zeroinitializer, align 16
+
+; AVX2-LABEL: @byte_sum_v32_i64(
+; AVX2: call <4 x i64> @llvm.x86.avx2.psad.bw(
+; AVX512-LABEL: @byte_sum_v32_i64(
+; AVX512: call <4 x i64> @llvm.x86.avx2.psad.bw(
+define i64 @byte_sum_v32_i64() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <32 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <32 x i8>, ptr %p, align 16
+ %z = zext <32 x i8> %wide.load to <32 x i64>
+ %add = add nsw <32 x i64> %z, %vec.phi
+ %index.next = add i64 %index, 32
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i64 @llvm.vector.reduce.add.v32i64(<32 x i64> %add)
+ ret i64 %ext
+}
+
+; AVX512-LABEL: @byte_sum_v64_i64(
+; AVX512: call <8 x i64> @llvm.x86.avx512.psad.bw.512(
+define i64 @byte_sum_v64_i64() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <64 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <64 x i8>, ptr %p, align 16
+ %z = zext <64 x i8> %wide.load to <64 x i64>
+ %add = add nsw <64 x i64> %z, %vec.phi
+ %index.next = add i64 %index, 64
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i64 @llvm.vector.reduce.add.v64i64(<64 x i64> %add)
+ ret i64 %ext
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
new file mode 100644
index 0000000000000..6acd8de044aa2
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+sse2 -S | FileCheck %s
+
+; Isolate X86PartialReduction::tryByteSumReplacement on a positive shape.
+
+ at a = global [1024 x i8] zeroinitializer, align 16
+
+; CHECK: call <2 x i64> @llvm.x86.sse2.psad.bw(
+define i32 @byte_sum_v16_i32() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <16 x i8>, ptr %p, align 16
+ %z = zext <16 x i8> %wide.load to <16 x i32>
+ %add = add nsw <16 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
+ ret i32 %ext
+}
>From 4f9d3918aab73b1f66b64a3bffbe2a775d46cf24 Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Thu, 6 Aug 2026 17:47:29 +0530
Subject: [PATCH 3/4] Merge opt tests and add x86-64-v2/v3/v4 coverage
---
llvm/test/CodeGen/X86/byte-sum.ll | 501 ++++++++++++++----
...x86-partial-reduction-byte-sum-debugloc.ll | 2 +-
...x86-partial-reduction-byte-sum-negative.ll | 49 --
...x86-partial-reduction-byte-sum-wide-i64.ll | 59 ---
.../X86/x86-partial-reduction-byte-sum.ll | 110 +++-
5 files changed, 516 insertions(+), 205 deletions(-)
delete mode 100644 llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll
delete mode 100644 llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll
diff --git a/llvm/test/CodeGen/X86/byte-sum.ll b/llvm/test/CodeGen/X86/byte-sum.ll
index 0a22fe31b46f7..85988357d55ac 100644
--- a/llvm/test/CodeGen/X86/byte-sum.ll
+++ b/llvm/test/CodeGen/X86/byte-sum.ll
@@ -1,7 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefixes=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE42
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512BW
; CodeGen tests for X86PartialReduction::tryByteSumReplacement.
; Positive cases should lower to vpsadbw; negative cases should not.
@@ -15,16 +16,17 @@ define i32 @byte_sum_v16_i32() nounwind {
; SSE2-LABEL: byte_sum_v16_i32:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB0_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm2
+; SSE2-NEXT: movdqa (%rcx,%rax), %xmm2
; SSE2-NEXT: psadbw %xmm0, %xmm2
; SSE2-NEXT: paddd %xmm2, %xmm1
; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB0_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm0, %xmm1
@@ -37,19 +39,46 @@ define i32 @byte_sum_v16_i32() nounwind {
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v16_i32:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB0_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: movdqa (%rcx,%rax), %xmm2
+; SSE42-NEXT: psadbw %xmm0, %xmm2
+; SSE42-NEXT: paddd %xmm2, %xmm1
+; SSE42-NEXT: addq $16, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB0_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: paddd %xmm0, %xmm0
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: movd %xmm0, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v16_i32:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB0_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm3
+; AVX2-NEXT: vpsadbw (%rcx,%rax), %xmm1, %xmm3
; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB0_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
@@ -66,15 +95,16 @@ define i32 @byte_sum_v16_i32() nounwind {
; AVX512BW-LABEL: byte_sum_v16_i32:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB0_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm2
+; AVX512BW-NEXT: vpsadbw (%rcx,%rax), %xmm1, %xmm2
; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB0_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
@@ -114,20 +144,21 @@ define i32 @byte_sum_v32_i32() nounwind {
; SSE2-LABEL: byte_sum_v32_i32:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB1_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm3
+; SSE2-NEXT: movdqa (%rcx,%rax), %xmm3
; SSE2-NEXT: psadbw %xmm0, %xmm3
; SSE2-NEXT: paddd %xmm3, %xmm1
-; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm3
+; SSE2-NEXT: movdqa 16(%rcx,%rax), %xmm3
; SSE2-NEXT: psadbw %xmm0, %xmm3
; SSE2-NEXT: paddd %xmm3, %xmm2
; SSE2-NEXT: addq $32, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB1_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm0, %xmm2
@@ -143,18 +174,52 @@ define i32 @byte_sum_v32_i32() nounwind {
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v32_i32:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB1_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: movdqa (%rcx,%rax), %xmm3
+; SSE42-NEXT: psadbw %xmm0, %xmm3
+; SSE42-NEXT: paddd %xmm3, %xmm1
+; SSE42-NEXT: movdqa 16(%rcx,%rax), %xmm3
+; SSE42-NEXT: psadbw %xmm0, %xmm3
+; SSE42-NEXT: paddd %xmm3, %xmm2
+; SSE42-NEXT: addq $32, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB1_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm0, %xmm2
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: paddd %xmm0, %xmm0
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: paddd %xmm2, %xmm0
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: movd %xmm0, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v32_i32:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB1_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %ymm0, %ymm2
+; AVX2-NEXT: vpsadbw (%rcx,%rax), %ymm0, %ymm2
; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
; AVX2-NEXT: addq $32, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB1_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm1
@@ -173,16 +238,17 @@ define i32 @byte_sum_v32_i32() nounwind {
; AVX512BW-LABEL: byte_sum_v32_i32:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB1_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %ymm1, %ymm3
+; AVX512BW-NEXT: vpsadbw (%rcx,%rax), %ymm1, %ymm3
; AVX512BW-NEXT: vpaddd %zmm2, %zmm3, %zmm2
; AVX512BW-NEXT: addq $32, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB1_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
@@ -223,7 +289,7 @@ define i32 @byte_sum_v64_i32() nounwind {
; SSE2-LABEL: byte_sum_v64_i32:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pxor %xmm4, %xmm4
@@ -232,19 +298,20 @@ define i32 @byte_sum_v64_i32() nounwind {
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB2_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm5
+; SSE2-NEXT: movdqa (%rcx,%rax), %xmm5
; SSE2-NEXT: psadbw %xmm3, %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm0
-; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm5
+; SSE2-NEXT: movdqa 16(%rcx,%rax), %xmm5
; SSE2-NEXT: psadbw %xmm3, %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm4
-; SSE2-NEXT: movdqa 1056(%rcx,%rax), %xmm5
+; SSE2-NEXT: movdqa 32(%rcx,%rax), %xmm5
; SSE2-NEXT: psadbw %xmm3, %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm2
-; SSE2-NEXT: movdqa 1072(%rcx,%rax), %xmm5
+; SSE2-NEXT: movdqa 48(%rcx,%rax), %xmm5
; SSE2-NEXT: psadbw %xmm3, %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm1
; SSE2-NEXT: addq $64, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB2_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm3, %xmm2
@@ -267,21 +334,70 @@ define i32 @byte_sum_v64_i32() nounwind {
; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v64_i32:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: pxor %xmm4, %xmm4
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB2_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: movdqa (%rcx,%rax), %xmm5
+; SSE42-NEXT: psadbw %xmm3, %xmm5
+; SSE42-NEXT: paddd %xmm5, %xmm0
+; SSE42-NEXT: movdqa 16(%rcx,%rax), %xmm5
+; SSE42-NEXT: psadbw %xmm3, %xmm5
+; SSE42-NEXT: paddd %xmm5, %xmm4
+; SSE42-NEXT: movdqa 32(%rcx,%rax), %xmm5
+; SSE42-NEXT: psadbw %xmm3, %xmm5
+; SSE42-NEXT: paddd %xmm5, %xmm2
+; SSE42-NEXT: movdqa 48(%rcx,%rax), %xmm5
+; SSE42-NEXT: psadbw %xmm3, %xmm5
+; SSE42-NEXT: paddd %xmm5, %xmm1
+; SSE42-NEXT: addq $64, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB2_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm3, %xmm2
+; SSE42-NEXT: pxor %xmm5, %xmm5
+; SSE42-NEXT: paddd %xmm5, %xmm5
+; SSE42-NEXT: paddd %xmm3, %xmm0
+; SSE42-NEXT: paddd %xmm3, %xmm1
+; SSE42-NEXT: paddd %xmm3, %xmm4
+; SSE42-NEXT: paddd %xmm5, %xmm4
+; SSE42-NEXT: paddd %xmm5, %xmm1
+; SSE42-NEXT: paddd %xmm4, %xmm1
+; SSE42-NEXT: paddd %xmm5, %xmm0
+; SSE42-NEXT: paddd %xmm2, %xmm5
+; SSE42-NEXT: paddd %xmm0, %xmm5
+; SSE42-NEXT: paddd %xmm1, %xmm5
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; SSE42-NEXT: paddd %xmm5, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v64_i32:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB2_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %ymm0, %ymm3
+; AVX2-NEXT: vpsadbw (%rcx,%rax), %ymm0, %ymm3
+; AVX2-NEXT: vpsadbw 32(%rcx,%rax), %ymm0, %ymm4
; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
-; AVX2-NEXT: vpsadbw 1056(%rcx,%rax), %ymm0, %ymm3
-; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpaddd %ymm2, %ymm4, %ymm2
; AVX2-NEXT: addq $64, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB2_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm2
@@ -303,15 +419,16 @@ define i32 @byte_sum_v64_i32() nounwind {
; AVX512BW-LABEL: byte_sum_v64_i32:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB2_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %zmm0, %zmm2
+; AVX512BW-NEXT: vpsadbw (%rcx,%rax), %zmm0, %zmm2
; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1
; AVX512BW-NEXT: addq $64, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB2_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm1
@@ -354,16 +471,17 @@ define i64 @byte_sum_v16_i64() nounwind {
; SSE2-LABEL: byte_sum_v16_i64:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB3_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm2
+; SSE2-NEXT: movdqa (%rcx,%rax), %xmm2
; SSE2-NEXT: psadbw %xmm0, %xmm2
; SSE2-NEXT: paddq %xmm2, %xmm1
; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB3_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddq %xmm0, %xmm1
@@ -377,19 +495,47 @@ define i64 @byte_sum_v16_i64() nounwind {
; SSE2-NEXT: movq %xmm0, %rax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v16_i64:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB3_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: movdqa (%rcx,%rax), %xmm2
+; SSE42-NEXT: psadbw %xmm0, %xmm2
+; SSE42-NEXT: paddq %xmm2, %xmm1
+; SSE42-NEXT: addq $16, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB3_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddq %xmm0, %xmm1
+; SSE42-NEXT: paddq %xmm0, %xmm0
+; SSE42-NEXT: movdqa %xmm0, %xmm2
+; SSE42-NEXT: paddq %xmm0, %xmm2
+; SSE42-NEXT: paddq %xmm0, %xmm2
+; SSE42-NEXT: paddq %xmm1, %xmm2
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; SSE42-NEXT: paddq %xmm2, %xmm0
+; SSE42-NEXT: movq %xmm0, %rax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v16_i64:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB3_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm3
+; AVX2-NEXT: vpsadbw (%rcx,%rax), %xmm1, %xmm3
; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm2
; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB3_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddq %ymm0, %ymm2, %ymm1
@@ -406,16 +552,17 @@ define i64 @byte_sum_v16_i64() nounwind {
; AVX512BW-LABEL: byte_sum_v16_i64:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB3_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm3
+; AVX512BW-NEXT: vpsadbw (%rcx,%rax), %xmm1, %xmm3
; AVX512BW-NEXT: vpaddq %zmm2, %zmm3, %zmm2
; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB3_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vpaddq %zmm0, %zmm2, %zmm0
@@ -454,7 +601,7 @@ define i32 @byte_sum_v16_i32_sext() nounwind {
; SSE2-LABEL: byte_sum_v16_i32_sext:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm3, %xmm3
@@ -462,7 +609,7 @@ define i32 @byte_sum_v16_i32_sext() nounwind {
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB4_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm4
+; SSE2-NEXT: movdqa (%rcx,%rax), %xmm4
; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
; SSE2-NEXT: psrad $24, %xmm6
@@ -478,6 +625,7 @@ define i32 @byte_sum_v16_i32_sext() nounwind {
; SSE2-NEXT: psrad $24, %xmm4
; SSE2-NEXT: paddd %xmm4, %xmm2
; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB4_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm3, %xmm0
@@ -490,20 +638,54 @@ define i32 @byte_sum_v16_i32_sext() nounwind {
; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v16_i32_sext:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB4_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: pmovsxbd 12(%rcx,%rax), %xmm4
+; SSE42-NEXT: paddd %xmm4, %xmm2
+; SSE42-NEXT: pmovsxbd 8(%rcx,%rax), %xmm4
+; SSE42-NEXT: paddd %xmm4, %xmm3
+; SSE42-NEXT: pmovsxbd (%rcx,%rax), %xmm4
+; SSE42-NEXT: pmovsxbd 4(%rcx,%rax), %xmm5
+; SSE42-NEXT: paddd %xmm4, %xmm0
+; SSE42-NEXT: paddd %xmm5, %xmm1
+; SSE42-NEXT: addq $16, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB4_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm3, %xmm0
+; SSE42-NEXT: paddd %xmm2, %xmm1
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v16_i32_sext:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB4_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpmovsxbd 1024(%rcx,%rax), %ymm2
+; AVX2-NEXT: vpmovsxbd (%rcx,%rax), %ymm2
+; AVX2-NEXT: vpmovsxbd 8(%rcx,%rax), %ymm3
; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpmovsxbd 1032(%rcx,%rax), %ymm2
-; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm3, %ymm0
; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB4_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
@@ -520,14 +702,15 @@ define i32 @byte_sum_v16_i32_sext() nounwind {
; AVX512BW-LABEL: byte_sum_v16_i32_sext:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB4_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpmovsxbd 1024(%rcx,%rax), %zmm1
+; AVX512BW-NEXT: vpmovsxbd (%rcx,%rax), %zmm1
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB4_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
@@ -567,7 +750,7 @@ define i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; SSE2-LABEL: word_sum_v16_i32:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: movq $-2048, %rax # imm = 0xF800
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pxor %xmm4, %xmm4
@@ -575,8 +758,8 @@ define i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB5_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqu 2048(%rdi,%rax), %xmm5
-; SSE2-NEXT: movdqu 2064(%rdi,%rax), %xmm6
+; SSE2-NEXT: movdqu (%rdi,%rax,2), %xmm5
+; SSE2-NEXT: movdqu 16(%rdi,%rax,2), %xmm6
; SSE2-NEXT: movdqa %xmm6, %xmm7
; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
; SSE2-NEXT: paddd %xmm7, %xmm3
@@ -587,7 +770,8 @@ define i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; SSE2-NEXT: paddd %xmm5, %xmm0
; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
; SSE2-NEXT: paddd %xmm6, %xmm4
-; SSE2-NEXT: addq $32, %rax
+; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB5_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm4, %xmm1
@@ -600,19 +784,52 @@ define i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: word_sum_v16_i32:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB5_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; SSE42-NEXT: paddd %xmm4, %xmm2
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; SSE42-NEXT: paddd %xmm4, %xmm3
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; SSE42-NEXT: paddd %xmm4, %xmm0
+; SSE42-NEXT: paddd %xmm5, %xmm1
+; SSE42-NEXT: addq $16, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB5_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm3, %xmm0
+; SSE42-NEXT: paddd %xmm2, %xmm1
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: word_sum_v16_i32:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-2048, %rax # imm = 0xF800
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB5_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
-; AVX2-NEXT: addq $32, %rax
+; AVX2-NEXT: vpaddd %ymm0, %ymm3, %ymm0
+; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB5_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
@@ -629,13 +846,14 @@ define i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; AVX512BW-LABEL: word_sum_v16_i32:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-2048, %rax # imm = 0xF800
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB5_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT: addq $32, %rax
+; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB5_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
@@ -674,39 +892,69 @@ middle.block:
define i16 @byte_sum_v16_i16() nounwind {
; SSE2-LABEL: byte_sum_v16_i16:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
-; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: xorl %eax, %eax
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB6_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm3
+; SSE2-NEXT: movdqa (%rcx,%rax), %xmm3
; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
-; SSE2-NEXT: paddw %xmm4, %xmm0
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; SSE2-NEXT: paddw %xmm4, %xmm1
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
; SSE2-NEXT: paddw %xmm3, %xmm2
; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB6_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddw %xmm2, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE2-NEXT: paddw %xmm0, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddw %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; SSE2-NEXT: paddw %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $16, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; SSE2-NEXT: paddw %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: paddw %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v16_i16:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB6_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: pmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; SSE42-NEXT: pmovzxbw {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; SSE42-NEXT: paddw %xmm2, %xmm0
+; SSE42-NEXT: paddw %xmm3, %xmm1
+; SSE42-NEXT: addq $16, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB6_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddw %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE42-NEXT: paddw %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: paddw %xmm1, %xmm0
+; SSE42-NEXT: movdqa %xmm0, %xmm1
+; SSE42-NEXT: psrld $16, %xmm1
+; SSE42-NEXT: paddw %xmm0, %xmm1
+; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v16_i16:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB6_1: # %vector.body
@@ -714,6 +962,7 @@ define i16 @byte_sum_v16_i16() nounwind {
; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
; AVX2-NEXT: vpaddw %ymm0, %ymm1, %ymm0
; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB6_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
@@ -732,7 +981,7 @@ define i16 @byte_sum_v16_i16() nounwind {
; AVX512BW-LABEL: byte_sum_v16_i16:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB6_1: # %vector.body
@@ -740,6 +989,7 @@ define i16 @byte_sum_v16_i16() nounwind {
; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
; AVX512BW-NEXT: vpaddw %ymm0, %ymm1, %ymm0
; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB6_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
@@ -780,22 +1030,23 @@ define i32 @dword_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; SSE2-LABEL: dword_sum_v16_i32:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movq $-4096, %rax # imm = 0xF000
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB7_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqu 4096(%rdi,%rax), %xmm4
+; SSE2-NEXT: movdqu (%rdi,%rax,4), %xmm4
; SSE2-NEXT: paddd %xmm4, %xmm0
-; SSE2-NEXT: movdqu 4112(%rdi,%rax), %xmm4
+; SSE2-NEXT: movdqu 16(%rdi,%rax,4), %xmm4
; SSE2-NEXT: paddd %xmm4, %xmm1
-; SSE2-NEXT: movdqu 4128(%rdi,%rax), %xmm4
+; SSE2-NEXT: movdqu 32(%rdi,%rax,4), %xmm4
; SSE2-NEXT: paddd %xmm4, %xmm3
-; SSE2-NEXT: movdqu 4144(%rdi,%rax), %xmm4
+; SSE2-NEXT: movdqu 48(%rdi,%rax,4), %xmm4
; SSE2-NEXT: paddd %xmm4, %xmm2
-; SSE2-NEXT: addq $64, %rax
+; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB7_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm3, %xmm0
@@ -808,17 +1059,50 @@ define i32 @dword_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: dword_sum_v16_i32:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB7_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: movdqu (%rdi,%rax,4), %xmm4
+; SSE42-NEXT: paddd %xmm4, %xmm0
+; SSE42-NEXT: movdqu 16(%rdi,%rax,4), %xmm4
+; SSE42-NEXT: paddd %xmm4, %xmm1
+; SSE42-NEXT: movdqu 32(%rdi,%rax,4), %xmm4
+; SSE42-NEXT: paddd %xmm4, %xmm3
+; SSE42-NEXT: movdqu 48(%rdi,%rax,4), %xmm4
+; SSE42-NEXT: paddd %xmm4, %xmm2
+; SSE42-NEXT: addq $16, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB7_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm3, %xmm0
+; SSE42-NEXT: paddd %xmm2, %xmm1
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: dword_sum_v16_i32:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-4096, %rax # imm = 0xF000
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB7_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpaddd 4096(%rdi,%rax), %ymm1, %ymm1
-; AVX2-NEXT: vpaddd 4128(%rdi,%rax), %ymm0, %ymm0
-; AVX2-NEXT: addq $64, %rax
+; AVX2-NEXT: vpaddd (%rdi,%rax,4), %ymm1, %ymm1
+; AVX2-NEXT: vpaddd 32(%rdi,%rax,4), %ymm0, %ymm0
+; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB7_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
@@ -835,12 +1119,13 @@ define i32 @dword_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; AVX512BW-LABEL: dword_sum_v16_i32:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-4096, %rax # imm = 0xF000
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB7_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpaddd 4096(%rdi,%rax), %zmm0, %zmm0
-; AVX512BW-NEXT: addq $64, %rax
+; AVX512BW-NEXT: vpaddd (%rdi,%rax,4), %zmm0, %zmm0
+; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB7_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
@@ -878,36 +1163,62 @@ middle.block:
define i32 @byte_sum_v8_i32() nounwind {
; SSE2-LABEL: byte_sum_v8_i32:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
-; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: xorl %eax, %eax
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB8_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
; SSE2-NEXT: movq {{.*#+}} xmm3 = mem[0],zero
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
-; SSE2-NEXT: paddd %xmm4, %xmm0
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; SSE2-NEXT: paddd %xmm4, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
; SSE2-NEXT: paddd %xmm3, %xmm2
; SSE2-NEXT: addq $8, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB8_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm2, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; SSE2-NEXT: paddd %xmm1, %xmm0
-; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v8_i32:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB8_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: pmovzxbd {{.*#+}} xmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; SSE42-NEXT: pmovzxbd {{.*#+}} xmm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; SSE42-NEXT: paddd %xmm2, %xmm0
+; SSE42-NEXT: paddd %xmm3, %xmm1
+; SSE42-NEXT: addq $8, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB8_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: movd %xmm0, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v8_i32:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB8_1: # %vector.body
@@ -915,6 +1226,7 @@ define i32 @byte_sum_v8_i32() nounwind {
; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX2-NEXT: addq $8, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB8_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
@@ -930,7 +1242,7 @@ define i32 @byte_sum_v8_i32() nounwind {
; AVX512BW-LABEL: byte_sum_v8_i32:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB8_1: # %vector.body
@@ -938,6 +1250,7 @@ define i32 @byte_sum_v8_i32() nounwind {
; AVX512BW-NEXT: vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
; AVX512BW-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX512BW-NEXT: addq $8, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB8_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll
index fcd757ba163ad..0501fb4f5e063 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+sse2 -S | FileCheck %s
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -S | FileCheck %s
; Verify that X86PartialReduction::tryByteSumReplacement carries the
; original add instruction's !dbg location onto the freshly emitted
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll
deleted file mode 100644
index d46edfd754514..0000000000000
--- a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll
+++ /dev/null
@@ -1,49 +0,0 @@
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+avx2 -S | FileCheck %s
-
-; Shapes that tryByteSumReplacement must not rewrite.
-
- at a = global [1024 x i8] zeroinitializer, align 16
-
-; CHECK-LABEL: @byte_sum_v8_i32
-; CHECK-NOT: psad.bw
-define i32 @byte_sum_v8_i32() nounwind {
-entry:
- br label %vector.body
-
-vector.body:
- %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
- %vec.phi = phi <8 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
- %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
- %wide.load = load <8 x i8>, ptr %p, align 8
- %z = zext <8 x i8> %wide.load to <8 x i32>
- %add = add nsw <8 x i32> %z, %vec.phi
- %index.next = add i64 %index, 8
- %cmp = icmp eq i64 %index.next, 1024
- br i1 %cmp, label %middle.block, label %vector.body
-
-middle.block:
- %ext = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %add)
- ret i32 %ext
-}
-
-; CHECK-LABEL: @byte_sum_v24_i32
-; CHECK-NOT: psad.bw
-define i32 @byte_sum_v24_i32() nounwind {
-entry:
- br label %vector.body
-
-vector.body:
- %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
- %vec.phi = phi <24 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
- %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
- %wide.load = load <24 x i8>, ptr %p, align 8
- %z = zext <24 x i8> %wide.load to <24 x i32>
- %add = add nsw <24 x i32> %z, %vec.phi
- %index.next = add i64 %index, 24
- %cmp = icmp eq i64 %index.next, 1024
- br i1 %cmp, label %middle.block, label %vector.body
-
-middle.block:
- %ext = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %add)
- ret i32 %ext
-}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll
deleted file mode 100644
index 23be1178bf767..0000000000000
--- a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll
+++ /dev/null
@@ -1,59 +0,0 @@
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+avx2 -S | FileCheck %s --check-prefix=AVX2
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+avx512bw -S | FileCheck %s --check-prefix=AVX512
-
-; Wider-VF i64-accumulator positive shapes for tryByteSumReplacement.
-; These exercise the PerSplitTy = <IntrinsicNumElts/8 x i64> branch where the
-; matcher must NOT bitcast the psadbw result back to i32.
-;
-; VF=32 i64 uses the AVX2 256-bit lane (one avx2.psad.bw call). On
-; +avx512bw the dispatch falls into the same AVX2 path because NumElts<64.
-;
-; VF=64 i64 uses the AVX-512BW 512-bit lane (one avx512.psad.bw.512 call).
-
- at a = global [1024 x i8] zeroinitializer, align 16
-
-; AVX2-LABEL: @byte_sum_v32_i64(
-; AVX2: call <4 x i64> @llvm.x86.avx2.psad.bw(
-; AVX512-LABEL: @byte_sum_v32_i64(
-; AVX512: call <4 x i64> @llvm.x86.avx2.psad.bw(
-define i64 @byte_sum_v32_i64() nounwind {
-entry:
- br label %vector.body
-
-vector.body:
- %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
- %vec.phi = phi <32 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
- %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
- %wide.load = load <32 x i8>, ptr %p, align 16
- %z = zext <32 x i8> %wide.load to <32 x i64>
- %add = add nsw <32 x i64> %z, %vec.phi
- %index.next = add i64 %index, 32
- %cmp = icmp eq i64 %index.next, 1024
- br i1 %cmp, label %middle.block, label %vector.body
-
-middle.block:
- %ext = call i64 @llvm.vector.reduce.add.v32i64(<32 x i64> %add)
- ret i64 %ext
-}
-
-; AVX512-LABEL: @byte_sum_v64_i64(
-; AVX512: call <8 x i64> @llvm.x86.avx512.psad.bw.512(
-define i64 @byte_sum_v64_i64() nounwind {
-entry:
- br label %vector.body
-
-vector.body:
- %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
- %vec.phi = phi <64 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
- %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
- %wide.load = load <64 x i8>, ptr %p, align 16
- %z = zext <64 x i8> %wide.load to <64 x i64>
- %add = add nsw <64 x i64> %z, %vec.phi
- %index.next = add i64 %index, 64
- %cmp = icmp eq i64 %index.next, 1024
- br i1 %cmp, label %middle.block, label %vector.body
-
-middle.block:
- %ext = call i64 @llvm.vector.reduce.add.v64i64(<64 x i64> %add)
- ret i64 %ext
-}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
index 6acd8de044aa2..9653ae7afd249 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
@@ -1,9 +1,15 @@
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+sse2 -S | FileCheck %s
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 -S | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 -S | FileCheck %s --check-prefixes=CHECK,AVX512
-; Isolate X86PartialReduction::tryByteSumReplacement on a positive shape.
+; Test X86PartialReduction::tryByteSumReplacement: positive and negative shapes.
@a = global [1024 x i8] zeroinitializer, align 16
+;; Positive cases -----------------------------------------------------------
+
+; CHECK-LABEL: @byte_sum_v16_i32(
; CHECK: call <2 x i64> @llvm.x86.sse2.psad.bw(
define i32 @byte_sum_v16_i32() nounwind {
entry:
@@ -24,3 +30,103 @@ middle.block:
%ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
ret i32 %ext
}
+
+; SSE-LABEL: @byte_sum_v32_i64(
+; SSE: call <2 x i64> @llvm.x86.sse2.psad.bw(
+; AVX2-LABEL: @byte_sum_v32_i64(
+; AVX2: call <4 x i64> @llvm.x86.avx2.psad.bw(
+; AVX512-LABEL: @byte_sum_v32_i64(
+; AVX512: call <4 x i64> @llvm.x86.avx2.psad.bw(
+define i64 @byte_sum_v32_i64() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <32 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <32 x i8>, ptr %p, align 16
+ %z = zext <32 x i8> %wide.load to <32 x i64>
+ %add = add nsw <32 x i64> %z, %vec.phi
+ %index.next = add i64 %index, 32
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i64 @llvm.vector.reduce.add.v32i64(<32 x i64> %add)
+ ret i64 %ext
+}
+
+; SSE-LABEL: @byte_sum_v64_i64(
+; SSE: call <2 x i64> @llvm.x86.sse2.psad.bw(
+; AVX2-LABEL: @byte_sum_v64_i64(
+; AVX2: call <4 x i64> @llvm.x86.avx2.psad.bw(
+; AVX512-LABEL: @byte_sum_v64_i64(
+; AVX512: call <8 x i64> @llvm.x86.avx512.psad.bw.512(
+define i64 @byte_sum_v64_i64() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <64 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <64 x i8>, ptr %p, align 16
+ %z = zext <64 x i8> %wide.load to <64 x i64>
+ %add = add nsw <64 x i64> %z, %vec.phi
+ %index.next = add i64 %index, 64
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i64 @llvm.vector.reduce.add.v64i64(<64 x i64> %add)
+ ret i64 %ext
+}
+
+;; Negative cases -----------------------------------------------------------
+
+; CHECK-LABEL: @byte_sum_v8_i32(
+; CHECK-NOT: psad.bw
+; CHECK: ret i32
+define i32 @byte_sum_v8_i32() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <8 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <8 x i8>, ptr %p, align 8
+ %z = zext <8 x i8> %wide.load to <8 x i32>
+ %add = add nsw <8 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 8
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %add)
+ ret i32 %ext
+}
+
+; CHECK-LABEL: @byte_sum_v24_i32(
+; CHECK-NOT: psad.bw
+; CHECK: ret i32
+define i32 @byte_sum_v24_i32() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <24 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <24 x i8>, ptr %p, align 8
+ %z = zext <24 x i8> %wide.load to <24 x i32>
+ %add = add nsw <24 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 24
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %add)
+ ret i32 %ext
+}
>From c615fbf404fc982ccd1d7646e3b4007e51155f72 Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Fri, 7 Aug 2026 17:31:46 +0530
Subject: [PATCH 4/4] Auto-generate CHECK lines for opt tests
---
.../X86/x86-partial-reduction-byte-sum.ll | 599 +++++++++++++++++-
1 file changed, 573 insertions(+), 26 deletions(-)
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
index 9653ae7afd249..2ce5aeeaf4d4f 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
@@ -1,17 +1,132 @@
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 -S | FileCheck %s --check-prefixes=CHECK,SSE
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 -S | FileCheck %s --check-prefixes=CHECK,AVX2
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 -S | FileCheck %s --check-prefixes=CHECK,AVX512
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -S | FileCheck %s --check-prefixes=SSE2
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 -S | FileCheck %s --check-prefixes=SSE42
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 -S | FileCheck %s --check-prefixes=AVX2
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 -S | FileCheck %s --check-prefixes=AVX512
; Test X86PartialReduction::tryByteSumReplacement: positive and negative shapes.
@a = global [1024 x i8] zeroinitializer, align 16
-;; Positive cases -----------------------------------------------------------
-; CHECK-LABEL: @byte_sum_v16_i32(
-; CHECK: call <2 x i64> @llvm.x86.sse2.psad.bw(
define i32 @byte_sum_v16_i32() nounwind {
+;
+; SSE2-LABEL: define i32 @byte_sum_v16_i32(
+; SSE2-SAME: ) #[[ATTR0:[0-9]+]] {
+; SSE2-NEXT: [[ENTRY:.*]]:
+; SSE2-NEXT: br label %[[VECTOR_BODY:.*]]
+; SSE2: [[VECTOR_BODY]]:
+; SSE2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SSE2-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; SSE2-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; SSE2-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[P]], align 16
+; SSE2-NEXT: [[TMP0:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> [[WIDE_LOAD]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP0]], <16 x i8> zeroinitializer)
+; SSE2-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to <4 x i32>
+; SSE2-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT: [[ADD]] = add nsw <16 x i32> [[TMP3]], [[VEC_PHI]]
+; SSE2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; SSE2-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SSE2-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; SSE2: [[MIDDLE_BLOCK]]:
+; SSE2-NEXT: [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[ADD]], <16 x i32> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[ADD]], [[RDX_SHUF]]
+; SSE2-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <16 x i32> [[BIN_RDX]], <16 x i32> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX2:%.*]] = add <16 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; SSE2-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <16 x i32> [[BIN_RDX2]], <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX4:%.*]] = add <16 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; SSE2-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <16 x i32> [[BIN_RDX4]], <16 x i32> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX6:%.*]] = add <16 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
+; SSE2-NEXT: [[TMP4:%.*]] = extractelement <16 x i32> [[BIN_RDX6]], i32 0
+; SSE2-NEXT: ret i32 [[TMP4]]
+;
+; SSE42-LABEL: define i32 @byte_sum_v16_i32(
+; SSE42-SAME: ) #[[ATTR0:[0-9]+]] {
+; SSE42-NEXT: [[ENTRY:.*]]:
+; SSE42-NEXT: br label %[[VECTOR_BODY:.*]]
+; SSE42: [[VECTOR_BODY]]:
+; SSE42-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SSE42-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; SSE42-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; SSE42-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[P]], align 16
+; SSE42-NEXT: [[TMP0:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> [[WIDE_LOAD]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP0]], <16 x i8> zeroinitializer)
+; SSE42-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to <4 x i32>
+; SSE42-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+; SSE42-NEXT: [[ADD]] = add nsw <16 x i32> [[TMP3]], [[VEC_PHI]]
+; SSE42-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; SSE42-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SSE42-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; SSE42: [[MIDDLE_BLOCK]]:
+; SSE42-NEXT: [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[ADD]], <16 x i32> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[ADD]], [[RDX_SHUF]]
+; SSE42-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <16 x i32> [[BIN_RDX]], <16 x i32> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX2:%.*]] = add <16 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; SSE42-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <16 x i32> [[BIN_RDX2]], <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX4:%.*]] = add <16 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; SSE42-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <16 x i32> [[BIN_RDX4]], <16 x i32> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX6:%.*]] = add <16 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
+; SSE42-NEXT: [[TMP4:%.*]] = extractelement <16 x i32> [[BIN_RDX6]], i32 0
+; SSE42-NEXT: ret i32 [[TMP4]]
+;
+; AVX2-LABEL: define i32 @byte_sum_v16_i32(
+; AVX2-SAME: ) #[[ATTR0:[0-9]+]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX2: [[VECTOR_BODY]]:
+; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[P]], align 16
+; AVX2-NEXT: [[TMP0:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> [[WIDE_LOAD]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP0]], <16 x i8> zeroinitializer)
+; AVX2-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to <4 x i32>
+; AVX2-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT: [[ADD]] = add nsw <16 x i32> [[TMP3]], [[VEC_PHI]]
+; AVX2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; AVX2-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX2-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; AVX2: [[MIDDLE_BLOCK]]:
+; AVX2-NEXT: [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[ADD]], <16 x i32> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[ADD]], [[RDX_SHUF]]
+; AVX2-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <16 x i32> [[BIN_RDX]], <16 x i32> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX2:%.*]] = add <16 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; AVX2-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <16 x i32> [[BIN_RDX2]], <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX4:%.*]] = add <16 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; AVX2-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <16 x i32> [[BIN_RDX4]], <16 x i32> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX6:%.*]] = add <16 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
+; AVX2-NEXT: [[TMP4:%.*]] = extractelement <16 x i32> [[BIN_RDX6]], i32 0
+; AVX2-NEXT: ret i32 [[TMP4]]
+;
+; AVX512-LABEL: define i32 @byte_sum_v16_i32(
+; AVX512-SAME: ) #[[ATTR0:[0-9]+]] {
+; AVX512-NEXT: [[ENTRY:.*]]:
+; AVX512-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX512: [[VECTOR_BODY]]:
+; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX512-NEXT: [[VEC_PHI:%.*]] = phi <16 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; AVX512-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <16 x i8>, ptr [[P]], align 16
+; AVX512-NEXT: [[TMP0:%.*]] = shufflevector <16 x i8> [[WIDE_LOAD]], <16 x i8> [[WIDE_LOAD]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP0]], <16 x i8> zeroinitializer)
+; AVX512-NEXT: [[TMP2:%.*]] = bitcast <2 x i64> [[TMP1]] to <4 x i32>
+; AVX512-NEXT: [[TMP3:%.*]] = shufflevector <4 x i32> [[TMP2]], <4 x i32> zeroinitializer, <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT: [[ADD]] = add nsw <16 x i32> [[TMP3]], [[VEC_PHI]]
+; AVX512-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 16
+; AVX512-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX512-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; AVX512: [[MIDDLE_BLOCK]]:
+; AVX512-NEXT: [[RDX_SHUF:%.*]] = shufflevector <16 x i32> [[ADD]], <16 x i32> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX:%.*]] = add <16 x i32> [[ADD]], [[RDX_SHUF]]
+; AVX512-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <16 x i32> [[BIN_RDX]], <16 x i32> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX2:%.*]] = add <16 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; AVX512-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <16 x i32> [[BIN_RDX2]], <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX4:%.*]] = add <16 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; AVX512-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <16 x i32> [[BIN_RDX4]], <16 x i32> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX6:%.*]] = add <16 x i32> [[BIN_RDX4]], [[RDX_SHUF5]]
+; AVX512-NEXT: [[TMP4:%.*]] = extractelement <16 x i32> [[BIN_RDX6]], i32 0
+; AVX512-NEXT: ret i32 [[TMP4]]
+;
entry:
br label %vector.body
@@ -31,13 +146,136 @@ middle.block:
ret i32 %ext
}
-; SSE-LABEL: @byte_sum_v32_i64(
-; SSE: call <2 x i64> @llvm.x86.sse2.psad.bw(
-; AVX2-LABEL: @byte_sum_v32_i64(
-; AVX2: call <4 x i64> @llvm.x86.avx2.psad.bw(
-; AVX512-LABEL: @byte_sum_v32_i64(
-; AVX512: call <4 x i64> @llvm.x86.avx2.psad.bw(
define i64 @byte_sum_v32_i64() nounwind {
+;
+;
+;
+; SSE2-LABEL: define i64 @byte_sum_v32_i64(
+; SSE2-SAME: ) #[[ATTR0]] {
+; SSE2-NEXT: [[ENTRY:.*]]:
+; SSE2-NEXT: br label %[[VECTOR_BODY:.*]]
+; SSE2: [[VECTOR_BODY]]:
+; SSE2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SSE2-NEXT: [[VEC_PHI:%.*]] = phi <32 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; SSE2-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; SSE2-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[P]], align 16
+; SSE2-NEXT: [[TMP0:%.*]] = shufflevector <32 x i8> [[WIDE_LOAD]], <32 x i8> [[WIDE_LOAD]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP0]], <16 x i8> zeroinitializer)
+; SSE2-NEXT: [[TMP2:%.*]] = shufflevector <32 x i8> [[WIDE_LOAD]], <32 x i8> [[WIDE_LOAD]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE2-NEXT: [[TMP3:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP2]], <16 x i8> zeroinitializer)
+; SSE2-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> zeroinitializer, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT: [[ADD]] = add nsw <32 x i64> [[TMP5]], [[VEC_PHI]]
+; SSE2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; SSE2-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SSE2-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; SSE2: [[MIDDLE_BLOCK]]:
+; SSE2-NEXT: [[RDX_SHUF:%.*]] = shufflevector <32 x i64> [[ADD]], <32 x i64> poison, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX:%.*]] = add <32 x i64> [[ADD]], [[RDX_SHUF]]
+; SSE2-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <32 x i64> [[BIN_RDX]], <32 x i64> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX2:%.*]] = add <32 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; SSE2-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <32 x i64> [[BIN_RDX2]], <32 x i64> poison, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX4:%.*]] = add <32 x i64> [[BIN_RDX2]], [[RDX_SHUF3]]
+; SSE2-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <32 x i64> [[BIN_RDX4]], <32 x i64> poison, <32 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX6:%.*]] = add <32 x i64> [[BIN_RDX4]], [[RDX_SHUF5]]
+; SSE2-NEXT: [[RDX_SHUF7:%.*]] = shufflevector <32 x i64> [[BIN_RDX6]], <32 x i64> poison, <32 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX8:%.*]] = add <32 x i64> [[BIN_RDX6]], [[RDX_SHUF7]]
+; SSE2-NEXT: [[TMP6:%.*]] = extractelement <32 x i64> [[BIN_RDX8]], i32 0
+; SSE2-NEXT: ret i64 [[TMP6]]
+;
+; SSE42-LABEL: define i64 @byte_sum_v32_i64(
+; SSE42-SAME: ) #[[ATTR0]] {
+; SSE42-NEXT: [[ENTRY:.*]]:
+; SSE42-NEXT: br label %[[VECTOR_BODY:.*]]
+; SSE42: [[VECTOR_BODY]]:
+; SSE42-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SSE42-NEXT: [[VEC_PHI:%.*]] = phi <32 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; SSE42-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; SSE42-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[P]], align 16
+; SSE42-NEXT: [[TMP0:%.*]] = shufflevector <32 x i8> [[WIDE_LOAD]], <32 x i8> [[WIDE_LOAD]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP0]], <16 x i8> zeroinitializer)
+; SSE42-NEXT: [[TMP2:%.*]] = shufflevector <32 x i8> [[WIDE_LOAD]], <32 x i8> [[WIDE_LOAD]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE42-NEXT: [[TMP3:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP2]], <16 x i8> zeroinitializer)
+; SSE42-NEXT: [[TMP4:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE42-NEXT: [[TMP5:%.*]] = shufflevector <4 x i64> [[TMP4]], <4 x i64> zeroinitializer, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+; SSE42-NEXT: [[ADD]] = add nsw <32 x i64> [[TMP5]], [[VEC_PHI]]
+; SSE42-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; SSE42-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SSE42-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; SSE42: [[MIDDLE_BLOCK]]:
+; SSE42-NEXT: [[RDX_SHUF:%.*]] = shufflevector <32 x i64> [[ADD]], <32 x i64> poison, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX:%.*]] = add <32 x i64> [[ADD]], [[RDX_SHUF]]
+; SSE42-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <32 x i64> [[BIN_RDX]], <32 x i64> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX2:%.*]] = add <32 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; SSE42-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <32 x i64> [[BIN_RDX2]], <32 x i64> poison, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX4:%.*]] = add <32 x i64> [[BIN_RDX2]], [[RDX_SHUF3]]
+; SSE42-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <32 x i64> [[BIN_RDX4]], <32 x i64> poison, <32 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX6:%.*]] = add <32 x i64> [[BIN_RDX4]], [[RDX_SHUF5]]
+; SSE42-NEXT: [[RDX_SHUF7:%.*]] = shufflevector <32 x i64> [[BIN_RDX6]], <32 x i64> poison, <32 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX8:%.*]] = add <32 x i64> [[BIN_RDX6]], [[RDX_SHUF7]]
+; SSE42-NEXT: [[TMP6:%.*]] = extractelement <32 x i64> [[BIN_RDX8]], i32 0
+; SSE42-NEXT: ret i64 [[TMP6]]
+;
+; AVX2-LABEL: define i64 @byte_sum_v32_i64(
+; AVX2-SAME: ) #[[ATTR0]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX2: [[VECTOR_BODY]]:
+; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[VEC_PHI:%.*]] = phi <32 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[P]], align 16
+; AVX2-NEXT: [[TMP0:%.*]] = shufflevector <32 x i8> [[WIDE_LOAD]], <32 x i8> [[WIDE_LOAD]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT: [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> [[TMP0]], <32 x i8> zeroinitializer)
+; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> zeroinitializer, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT: [[ADD]] = add nsw <32 x i64> [[TMP2]], [[VEC_PHI]]
+; AVX2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; AVX2-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX2-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; AVX2: [[MIDDLE_BLOCK]]:
+; AVX2-NEXT: [[RDX_SHUF:%.*]] = shufflevector <32 x i64> [[ADD]], <32 x i64> poison, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX:%.*]] = add <32 x i64> [[ADD]], [[RDX_SHUF]]
+; AVX2-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <32 x i64> [[BIN_RDX]], <32 x i64> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX2:%.*]] = add <32 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; AVX2-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <32 x i64> [[BIN_RDX2]], <32 x i64> poison, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX4:%.*]] = add <32 x i64> [[BIN_RDX2]], [[RDX_SHUF3]]
+; AVX2-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <32 x i64> [[BIN_RDX4]], <32 x i64> poison, <32 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX6:%.*]] = add <32 x i64> [[BIN_RDX4]], [[RDX_SHUF5]]
+; AVX2-NEXT: [[RDX_SHUF7:%.*]] = shufflevector <32 x i64> [[BIN_RDX6]], <32 x i64> poison, <32 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX8:%.*]] = add <32 x i64> [[BIN_RDX6]], [[RDX_SHUF7]]
+; AVX2-NEXT: [[TMP3:%.*]] = extractelement <32 x i64> [[BIN_RDX8]], i32 0
+; AVX2-NEXT: ret i64 [[TMP3]]
+;
+; AVX512-LABEL: define i64 @byte_sum_v32_i64(
+; AVX512-SAME: ) #[[ATTR0]] {
+; AVX512-NEXT: [[ENTRY:.*]]:
+; AVX512-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX512: [[VECTOR_BODY]]:
+; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX512-NEXT: [[VEC_PHI:%.*]] = phi <32 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; AVX512-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <32 x i8>, ptr [[P]], align 16
+; AVX512-NEXT: [[TMP0:%.*]] = shufflevector <32 x i8> [[WIDE_LOAD]], <32 x i8> [[WIDE_LOAD]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX512-NEXT: [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> [[TMP0]], <32 x i8> zeroinitializer)
+; AVX512-NEXT: [[TMP2:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> zeroinitializer, <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7, i32 4, i32 5, i32 6, i32 7>
+; AVX512-NEXT: [[ADD]] = add nsw <32 x i64> [[TMP2]], [[VEC_PHI]]
+; AVX512-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 32
+; AVX512-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX512-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; AVX512: [[MIDDLE_BLOCK]]:
+; AVX512-NEXT: [[RDX_SHUF:%.*]] = shufflevector <32 x i64> [[ADD]], <32 x i64> poison, <32 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX:%.*]] = add <32 x i64> [[ADD]], [[RDX_SHUF]]
+; AVX512-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <32 x i64> [[BIN_RDX]], <32 x i64> poison, <32 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX2:%.*]] = add <32 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; AVX512-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <32 x i64> [[BIN_RDX2]], <32 x i64> poison, <32 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX4:%.*]] = add <32 x i64> [[BIN_RDX2]], [[RDX_SHUF3]]
+; AVX512-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <32 x i64> [[BIN_RDX4]], <32 x i64> poison, <32 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX6:%.*]] = add <32 x i64> [[BIN_RDX4]], [[RDX_SHUF5]]
+; AVX512-NEXT: [[RDX_SHUF7:%.*]] = shufflevector <32 x i64> [[BIN_RDX6]], <32 x i64> poison, <32 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX8:%.*]] = add <32 x i64> [[BIN_RDX6]], [[RDX_SHUF7]]
+; AVX512-NEXT: [[TMP3:%.*]] = extractelement <32 x i64> [[BIN_RDX8]], i32 0
+; AVX512-NEXT: ret i64 [[TMP3]]
+;
entry:
br label %vector.body
@@ -57,13 +295,159 @@ middle.block:
ret i64 %ext
}
-; SSE-LABEL: @byte_sum_v64_i64(
-; SSE: call <2 x i64> @llvm.x86.sse2.psad.bw(
-; AVX2-LABEL: @byte_sum_v64_i64(
-; AVX2: call <4 x i64> @llvm.x86.avx2.psad.bw(
-; AVX512-LABEL: @byte_sum_v64_i64(
-; AVX512: call <8 x i64> @llvm.x86.avx512.psad.bw.512(
define i64 @byte_sum_v64_i64() nounwind {
+;
+;
+;
+; SSE2-LABEL: define i64 @byte_sum_v64_i64(
+; SSE2-SAME: ) #[[ATTR0]] {
+; SSE2-NEXT: [[ENTRY:.*]]:
+; SSE2-NEXT: br label %[[VECTOR_BODY:.*]]
+; SSE2: [[VECTOR_BODY]]:
+; SSE2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SSE2-NEXT: [[VEC_PHI:%.*]] = phi <64 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; SSE2-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; SSE2-NEXT: [[WIDE_LOAD:%.*]] = load <64 x i8>, ptr [[P]], align 16
+; SSE2-NEXT: [[TMP0:%.*]] = shufflevector <64 x i8> [[WIDE_LOAD]], <64 x i8> [[WIDE_LOAD]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP0]], <16 x i8> zeroinitializer)
+; SSE2-NEXT: [[TMP2:%.*]] = shufflevector <64 x i8> [[WIDE_LOAD]], <64 x i8> [[WIDE_LOAD]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE2-NEXT: [[TMP3:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP2]], <16 x i8> zeroinitializer)
+; SSE2-NEXT: [[TMP4:%.*]] = shufflevector <64 x i8> [[WIDE_LOAD]], <64 x i8> [[WIDE_LOAD]], <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; SSE2-NEXT: [[TMP5:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP4]], <16 x i8> zeroinitializer)
+; SSE2-NEXT: [[TMP6:%.*]] = shufflevector <64 x i8> [[WIDE_LOAD]], <64 x i8> [[WIDE_LOAD]], <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; SSE2-NEXT: [[TMP7:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP6]], <16 x i8> zeroinitializer)
+; SSE2-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE2-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE2-NEXT: [[TMP11:%.*]] = shufflevector <8 x i64> [[TMP10]], <8 x i64> zeroinitializer, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE2-NEXT: [[ADD]] = add nsw <64 x i64> [[TMP11]], [[VEC_PHI]]
+; SSE2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 64
+; SSE2-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SSE2-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; SSE2: [[MIDDLE_BLOCK]]:
+; SSE2-NEXT: [[RDX_SHUF:%.*]] = shufflevector <64 x i64> [[ADD]], <64 x i64> poison, <64 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX:%.*]] = add <64 x i64> [[ADD]], [[RDX_SHUF]]
+; SSE2-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <64 x i64> [[BIN_RDX]], <64 x i64> poison, <64 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX2:%.*]] = add <64 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; SSE2-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <64 x i64> [[BIN_RDX2]], <64 x i64> poison, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX4:%.*]] = add <64 x i64> [[BIN_RDX2]], [[RDX_SHUF3]]
+; SSE2-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <64 x i64> [[BIN_RDX4]], <64 x i64> poison, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX6:%.*]] = add <64 x i64> [[BIN_RDX4]], [[RDX_SHUF5]]
+; SSE2-NEXT: [[RDX_SHUF7:%.*]] = shufflevector <64 x i64> [[BIN_RDX6]], <64 x i64> poison, <64 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX8:%.*]] = add <64 x i64> [[BIN_RDX6]], [[RDX_SHUF7]]
+; SSE2-NEXT: [[RDX_SHUF9:%.*]] = shufflevector <64 x i64> [[BIN_RDX8]], <64 x i64> poison, <64 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX10:%.*]] = add <64 x i64> [[BIN_RDX8]], [[RDX_SHUF9]]
+; SSE2-NEXT: [[TMP12:%.*]] = extractelement <64 x i64> [[BIN_RDX10]], i32 0
+; SSE2-NEXT: ret i64 [[TMP12]]
+;
+; SSE42-LABEL: define i64 @byte_sum_v64_i64(
+; SSE42-SAME: ) #[[ATTR0]] {
+; SSE42-NEXT: [[ENTRY:.*]]:
+; SSE42-NEXT: br label %[[VECTOR_BODY:.*]]
+; SSE42: [[VECTOR_BODY]]:
+; SSE42-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SSE42-NEXT: [[VEC_PHI:%.*]] = phi <64 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; SSE42-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; SSE42-NEXT: [[WIDE_LOAD:%.*]] = load <64 x i8>, ptr [[P]], align 16
+; SSE42-NEXT: [[TMP0:%.*]] = shufflevector <64 x i8> [[WIDE_LOAD]], <64 x i8> [[WIDE_LOAD]], <16 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT: [[TMP1:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP0]], <16 x i8> zeroinitializer)
+; SSE42-NEXT: [[TMP2:%.*]] = shufflevector <64 x i8> [[WIDE_LOAD]], <64 x i8> [[WIDE_LOAD]], <16 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; SSE42-NEXT: [[TMP3:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP2]], <16 x i8> zeroinitializer)
+; SSE42-NEXT: [[TMP4:%.*]] = shufflevector <64 x i8> [[WIDE_LOAD]], <64 x i8> [[WIDE_LOAD]], <16 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47>
+; SSE42-NEXT: [[TMP5:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP4]], <16 x i8> zeroinitializer)
+; SSE42-NEXT: [[TMP6:%.*]] = shufflevector <64 x i8> [[WIDE_LOAD]], <64 x i8> [[WIDE_LOAD]], <16 x i32> <i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; SSE42-NEXT: [[TMP7:%.*]] = call <2 x i64> @llvm.x86.sse2.psad.bw(<16 x i8> [[TMP6]], <16 x i8> zeroinitializer)
+; SSE42-NEXT: [[TMP8:%.*]] = shufflevector <2 x i64> [[TMP1]], <2 x i64> [[TMP3]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE42-NEXT: [[TMP9:%.*]] = shufflevector <2 x i64> [[TMP5]], <2 x i64> [[TMP7]], <4 x i32> <i32 0, i32 1, i32 2, i32 3>
+; SSE42-NEXT: [[TMP10:%.*]] = shufflevector <4 x i64> [[TMP8]], <4 x i64> [[TMP9]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; SSE42-NEXT: [[TMP11:%.*]] = shufflevector <8 x i64> [[TMP10]], <8 x i64> zeroinitializer, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; SSE42-NEXT: [[ADD]] = add nsw <64 x i64> [[TMP11]], [[VEC_PHI]]
+; SSE42-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 64
+; SSE42-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SSE42-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; SSE42: [[MIDDLE_BLOCK]]:
+; SSE42-NEXT: [[RDX_SHUF:%.*]] = shufflevector <64 x i64> [[ADD]], <64 x i64> poison, <64 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX:%.*]] = add <64 x i64> [[ADD]], [[RDX_SHUF]]
+; SSE42-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <64 x i64> [[BIN_RDX]], <64 x i64> poison, <64 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX2:%.*]] = add <64 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; SSE42-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <64 x i64> [[BIN_RDX2]], <64 x i64> poison, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX4:%.*]] = add <64 x i64> [[BIN_RDX2]], [[RDX_SHUF3]]
+; SSE42-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <64 x i64> [[BIN_RDX4]], <64 x i64> poison, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX6:%.*]] = add <64 x i64> [[BIN_RDX4]], [[RDX_SHUF5]]
+; SSE42-NEXT: [[RDX_SHUF7:%.*]] = shufflevector <64 x i64> [[BIN_RDX6]], <64 x i64> poison, <64 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX8:%.*]] = add <64 x i64> [[BIN_RDX6]], [[RDX_SHUF7]]
+; SSE42-NEXT: [[RDX_SHUF9:%.*]] = shufflevector <64 x i64> [[BIN_RDX8]], <64 x i64> poison, <64 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX10:%.*]] = add <64 x i64> [[BIN_RDX8]], [[RDX_SHUF9]]
+; SSE42-NEXT: [[TMP12:%.*]] = extractelement <64 x i64> [[BIN_RDX10]], i32 0
+; SSE42-NEXT: ret i64 [[TMP12]]
+;
+; AVX2-LABEL: define i64 @byte_sum_v64_i64(
+; AVX2-SAME: ) #[[ATTR0]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX2: [[VECTOR_BODY]]:
+; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[VEC_PHI:%.*]] = phi <64 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <64 x i8>, ptr [[P]], align 16
+; AVX2-NEXT: [[TMP0:%.*]] = shufflevector <64 x i8> [[WIDE_LOAD]], <64 x i8> [[WIDE_LOAD]], <32 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31>
+; AVX2-NEXT: [[TMP1:%.*]] = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> [[TMP0]], <32 x i8> zeroinitializer)
+; AVX2-NEXT: [[TMP2:%.*]] = shufflevector <64 x i8> [[WIDE_LOAD]], <64 x i8> [[WIDE_LOAD]], <32 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX2-NEXT: [[TMP3:%.*]] = call <4 x i64> @llvm.x86.avx2.psad.bw(<32 x i8> [[TMP2]], <32 x i8> zeroinitializer)
+; AVX2-NEXT: [[TMP4:%.*]] = shufflevector <4 x i64> [[TMP1]], <4 x i64> [[TMP3]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7>
+; AVX2-NEXT: [[TMP5:%.*]] = shufflevector <8 x i64> [[TMP4]], <8 x i64> zeroinitializer, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX2-NEXT: [[ADD]] = add nsw <64 x i64> [[TMP5]], [[VEC_PHI]]
+; AVX2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 64
+; AVX2-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX2-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; AVX2: [[MIDDLE_BLOCK]]:
+; AVX2-NEXT: [[RDX_SHUF:%.*]] = shufflevector <64 x i64> [[ADD]], <64 x i64> poison, <64 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX:%.*]] = add <64 x i64> [[ADD]], [[RDX_SHUF]]
+; AVX2-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <64 x i64> [[BIN_RDX]], <64 x i64> poison, <64 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX2:%.*]] = add <64 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; AVX2-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <64 x i64> [[BIN_RDX2]], <64 x i64> poison, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX4:%.*]] = add <64 x i64> [[BIN_RDX2]], [[RDX_SHUF3]]
+; AVX2-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <64 x i64> [[BIN_RDX4]], <64 x i64> poison, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX6:%.*]] = add <64 x i64> [[BIN_RDX4]], [[RDX_SHUF5]]
+; AVX2-NEXT: [[RDX_SHUF7:%.*]] = shufflevector <64 x i64> [[BIN_RDX6]], <64 x i64> poison, <64 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX8:%.*]] = add <64 x i64> [[BIN_RDX6]], [[RDX_SHUF7]]
+; AVX2-NEXT: [[RDX_SHUF9:%.*]] = shufflevector <64 x i64> [[BIN_RDX8]], <64 x i64> poison, <64 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX10:%.*]] = add <64 x i64> [[BIN_RDX8]], [[RDX_SHUF9]]
+; AVX2-NEXT: [[TMP6:%.*]] = extractelement <64 x i64> [[BIN_RDX10]], i32 0
+; AVX2-NEXT: ret i64 [[TMP6]]
+;
+; AVX512-LABEL: define i64 @byte_sum_v64_i64(
+; AVX512-SAME: ) #[[ATTR0]] {
+; AVX512-NEXT: [[ENTRY:.*]]:
+; AVX512-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX512: [[VECTOR_BODY]]:
+; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX512-NEXT: [[VEC_PHI:%.*]] = phi <64 x i64> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; AVX512-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <64 x i8>, ptr [[P]], align 16
+; AVX512-NEXT: [[TMP0:%.*]] = shufflevector <64 x i8> [[WIDE_LOAD]], <64 x i8> [[WIDE_LOAD]], <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63>
+; AVX512-NEXT: [[TMP1:%.*]] = call <8 x i64> @llvm.x86.avx512.psad.bw.512(<64 x i8> [[TMP0]], <64 x i8> zeroinitializer)
+; AVX512-NEXT: [[TMP2:%.*]] = shufflevector <8 x i64> [[TMP1]], <8 x i64> zeroinitializer, <64 x i32> <i32 0, i32 1, i32 2, i32 3, i32 4, i32 5, i32 6, i32 7, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15>
+; AVX512-NEXT: [[ADD]] = add nsw <64 x i64> [[TMP2]], [[VEC_PHI]]
+; AVX512-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 64
+; AVX512-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX512-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; AVX512: [[MIDDLE_BLOCK]]:
+; AVX512-NEXT: [[RDX_SHUF:%.*]] = shufflevector <64 x i64> [[ADD]], <64 x i64> poison, <64 x i32> <i32 32, i32 33, i32 34, i32 35, i32 36, i32 37, i32 38, i32 39, i32 40, i32 41, i32 42, i32 43, i32 44, i32 45, i32 46, i32 47, i32 48, i32 49, i32 50, i32 51, i32 52, i32 53, i32 54, i32 55, i32 56, i32 57, i32 58, i32 59, i32 60, i32 61, i32 62, i32 63, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX:%.*]] = add <64 x i64> [[ADD]], [[RDX_SHUF]]
+; AVX512-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <64 x i64> [[BIN_RDX]], <64 x i64> poison, <64 x i32> <i32 16, i32 17, i32 18, i32 19, i32 20, i32 21, i32 22, i32 23, i32 24, i32 25, i32 26, i32 27, i32 28, i32 29, i32 30, i32 31, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX2:%.*]] = add <64 x i64> [[BIN_RDX]], [[RDX_SHUF1]]
+; AVX512-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <64 x i64> [[BIN_RDX2]], <64 x i64> poison, <64 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX4:%.*]] = add <64 x i64> [[BIN_RDX2]], [[RDX_SHUF3]]
+; AVX512-NEXT: [[RDX_SHUF5:%.*]] = shufflevector <64 x i64> [[BIN_RDX4]], <64 x i64> poison, <64 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX6:%.*]] = add <64 x i64> [[BIN_RDX4]], [[RDX_SHUF5]]
+; AVX512-NEXT: [[RDX_SHUF7:%.*]] = shufflevector <64 x i64> [[BIN_RDX6]], <64 x i64> poison, <64 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX8:%.*]] = add <64 x i64> [[BIN_RDX6]], [[RDX_SHUF7]]
+; AVX512-NEXT: [[RDX_SHUF9:%.*]] = shufflevector <64 x i64> [[BIN_RDX8]], <64 x i64> poison, <64 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX10:%.*]] = add <64 x i64> [[BIN_RDX8]], [[RDX_SHUF9]]
+; AVX512-NEXT: [[TMP3:%.*]] = extractelement <64 x i64> [[BIN_RDX10]], i32 0
+; AVX512-NEXT: ret i64 [[TMP3]]
+;
entry:
br label %vector.body
@@ -83,12 +467,105 @@ middle.block:
ret i64 %ext
}
-;; Negative cases -----------------------------------------------------------
-; CHECK-LABEL: @byte_sum_v8_i32(
-; CHECK-NOT: psad.bw
-; CHECK: ret i32
define i32 @byte_sum_v8_i32() nounwind {
+;
+; SSE2-LABEL: define i32 @byte_sum_v8_i32(
+; SSE2-SAME: ) #[[ATTR0]] {
+; SSE2-NEXT: [[ENTRY:.*]]:
+; SSE2-NEXT: br label %[[VECTOR_BODY:.*]]
+; SSE2: [[VECTOR_BODY]]:
+; SSE2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SSE2-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; SSE2-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; SSE2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[P]], align 8
+; SSE2-NEXT: [[Z:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
+; SSE2-NEXT: [[ADD]] = add nsw <8 x i32> [[Z]], [[VEC_PHI]]
+; SSE2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; SSE2-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SSE2-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; SSE2: [[MIDDLE_BLOCK]]:
+; SSE2-NEXT: [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[ADD]], <8 x i32> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[ADD]], [[RDX_SHUF]]
+; SSE2-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX2:%.*]] = add <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; SSE2-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE2-NEXT: [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; SSE2-NEXT: [[TMP0:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; SSE2-NEXT: ret i32 [[TMP0]]
+;
+; SSE42-LABEL: define i32 @byte_sum_v8_i32(
+; SSE42-SAME: ) #[[ATTR0]] {
+; SSE42-NEXT: [[ENTRY:.*]]:
+; SSE42-NEXT: br label %[[VECTOR_BODY:.*]]
+; SSE42: [[VECTOR_BODY]]:
+; SSE42-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SSE42-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; SSE42-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; SSE42-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[P]], align 8
+; SSE42-NEXT: [[Z:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
+; SSE42-NEXT: [[ADD]] = add nsw <8 x i32> [[Z]], [[VEC_PHI]]
+; SSE42-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; SSE42-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SSE42-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; SSE42: [[MIDDLE_BLOCK]]:
+; SSE42-NEXT: [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[ADD]], <8 x i32> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[ADD]], [[RDX_SHUF]]
+; SSE42-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX2:%.*]] = add <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; SSE42-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; SSE42-NEXT: [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; SSE42-NEXT: [[TMP0:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; SSE42-NEXT: ret i32 [[TMP0]]
+;
+; AVX2-LABEL: define i32 @byte_sum_v8_i32(
+; AVX2-SAME: ) #[[ATTR0]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX2: [[VECTOR_BODY]]:
+; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[P]], align 8
+; AVX2-NEXT: [[Z:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
+; AVX2-NEXT: [[ADD]] = add nsw <8 x i32> [[Z]], [[VEC_PHI]]
+; AVX2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; AVX2-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX2-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; AVX2: [[MIDDLE_BLOCK]]:
+; AVX2-NEXT: [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[ADD]], <8 x i32> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[ADD]], [[RDX_SHUF]]
+; AVX2-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX2:%.*]] = add <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; AVX2-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX2-NEXT: [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; AVX2-NEXT: [[TMP0:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; AVX2-NEXT: ret i32 [[TMP0]]
+;
+; AVX512-LABEL: define i32 @byte_sum_v8_i32(
+; AVX512-SAME: ) #[[ATTR0]] {
+; AVX512-NEXT: [[ENTRY:.*]]:
+; AVX512-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX512: [[VECTOR_BODY]]:
+; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX512-NEXT: [[VEC_PHI:%.*]] = phi <8 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; AVX512-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <8 x i8>, ptr [[P]], align 8
+; AVX512-NEXT: [[Z:%.*]] = zext <8 x i8> [[WIDE_LOAD]] to <8 x i32>
+; AVX512-NEXT: [[ADD]] = add nsw <8 x i32> [[Z]], [[VEC_PHI]]
+; AVX512-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 8
+; AVX512-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX512-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; AVX512: [[MIDDLE_BLOCK]]:
+; AVX512-NEXT: [[RDX_SHUF:%.*]] = shufflevector <8 x i32> [[ADD]], <8 x i32> poison, <8 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX:%.*]] = add <8 x i32> [[ADD]], [[RDX_SHUF]]
+; AVX512-NEXT: [[RDX_SHUF1:%.*]] = shufflevector <8 x i32> [[BIN_RDX]], <8 x i32> poison, <8 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX2:%.*]] = add <8 x i32> [[BIN_RDX]], [[RDX_SHUF1]]
+; AVX512-NEXT: [[RDX_SHUF3:%.*]] = shufflevector <8 x i32> [[BIN_RDX2]], <8 x i32> poison, <8 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+; AVX512-NEXT: [[BIN_RDX4:%.*]] = add <8 x i32> [[BIN_RDX2]], [[RDX_SHUF3]]
+; AVX512-NEXT: [[TMP0:%.*]] = extractelement <8 x i32> [[BIN_RDX4]], i32 0
+; AVX512-NEXT: ret i32 [[TMP0]]
+;
entry:
br label %vector.body
@@ -108,10 +585,80 @@ middle.block:
ret i32 %ext
}
-; CHECK-LABEL: @byte_sum_v24_i32(
-; CHECK-NOT: psad.bw
-; CHECK: ret i32
define i32 @byte_sum_v24_i32() nounwind {
+;
+; SSE2-LABEL: define i32 @byte_sum_v24_i32(
+; SSE2-SAME: ) #[[ATTR0]] {
+; SSE2-NEXT: [[ENTRY:.*]]:
+; SSE2-NEXT: br label %[[VECTOR_BODY:.*]]
+; SSE2: [[VECTOR_BODY]]:
+; SSE2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SSE2-NEXT: [[VEC_PHI:%.*]] = phi <24 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; SSE2-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; SSE2-NEXT: [[WIDE_LOAD:%.*]] = load <24 x i8>, ptr [[P]], align 8
+; SSE2-NEXT: [[Z:%.*]] = zext <24 x i8> [[WIDE_LOAD]] to <24 x i32>
+; SSE2-NEXT: [[ADD]] = add nsw <24 x i32> [[Z]], [[VEC_PHI]]
+; SSE2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 24
+; SSE2-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SSE2-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; SSE2: [[MIDDLE_BLOCK]]:
+; SSE2-NEXT: [[EXT:%.*]] = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> [[ADD]])
+; SSE2-NEXT: ret i32 [[EXT]]
+;
+; SSE42-LABEL: define i32 @byte_sum_v24_i32(
+; SSE42-SAME: ) #[[ATTR0]] {
+; SSE42-NEXT: [[ENTRY:.*]]:
+; SSE42-NEXT: br label %[[VECTOR_BODY:.*]]
+; SSE42: [[VECTOR_BODY]]:
+; SSE42-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; SSE42-NEXT: [[VEC_PHI:%.*]] = phi <24 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; SSE42-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; SSE42-NEXT: [[WIDE_LOAD:%.*]] = load <24 x i8>, ptr [[P]], align 8
+; SSE42-NEXT: [[Z:%.*]] = zext <24 x i8> [[WIDE_LOAD]] to <24 x i32>
+; SSE42-NEXT: [[ADD]] = add nsw <24 x i32> [[Z]], [[VEC_PHI]]
+; SSE42-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 24
+; SSE42-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; SSE42-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; SSE42: [[MIDDLE_BLOCK]]:
+; SSE42-NEXT: [[EXT:%.*]] = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> [[ADD]])
+; SSE42-NEXT: ret i32 [[EXT]]
+;
+; AVX2-LABEL: define i32 @byte_sum_v24_i32(
+; AVX2-SAME: ) #[[ATTR0]] {
+; AVX2-NEXT: [[ENTRY:.*]]:
+; AVX2-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX2: [[VECTOR_BODY]]:
+; AVX2-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[VEC_PHI:%.*]] = phi <24 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; AVX2-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; AVX2-NEXT: [[WIDE_LOAD:%.*]] = load <24 x i8>, ptr [[P]], align 8
+; AVX2-NEXT: [[Z:%.*]] = zext <24 x i8> [[WIDE_LOAD]] to <24 x i32>
+; AVX2-NEXT: [[ADD]] = add nsw <24 x i32> [[Z]], [[VEC_PHI]]
+; AVX2-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 24
+; AVX2-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX2-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; AVX2: [[MIDDLE_BLOCK]]:
+; AVX2-NEXT: [[EXT:%.*]] = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> [[ADD]])
+; AVX2-NEXT: ret i32 [[EXT]]
+;
+; AVX512-LABEL: define i32 @byte_sum_v24_i32(
+; AVX512-SAME: ) #[[ATTR0]] {
+; AVX512-NEXT: [[ENTRY:.*]]:
+; AVX512-NEXT: br label %[[VECTOR_BODY:.*]]
+; AVX512: [[VECTOR_BODY]]:
+; AVX512-NEXT: [[INDEX:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[INDEX_NEXT:%.*]], %[[VECTOR_BODY]] ]
+; AVX512-NEXT: [[VEC_PHI:%.*]] = phi <24 x i32> [ zeroinitializer, %[[ENTRY]] ], [ [[ADD:%.*]], %[[VECTOR_BODY]] ]
+; AVX512-NEXT: [[P:%.*]] = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 [[INDEX]]
+; AVX512-NEXT: [[WIDE_LOAD:%.*]] = load <24 x i8>, ptr [[P]], align 8
+; AVX512-NEXT: [[Z:%.*]] = zext <24 x i8> [[WIDE_LOAD]] to <24 x i32>
+; AVX512-NEXT: [[ADD]] = add nsw <24 x i32> [[Z]], [[VEC_PHI]]
+; AVX512-NEXT: [[INDEX_NEXT]] = add i64 [[INDEX]], 24
+; AVX512-NEXT: [[CMP:%.*]] = icmp eq i64 [[INDEX_NEXT]], 1024
+; AVX512-NEXT: br i1 [[CMP]], label %[[MIDDLE_BLOCK:.*]], label %[[VECTOR_BODY]]
+; AVX512: [[MIDDLE_BLOCK]]:
+; AVX512-NEXT: [[EXT:%.*]] = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> [[ADD]])
+; AVX512-NEXT: ret i32 [[EXT]]
+;
entry:
br label %vector.body
More information about the llvm-commits
mailing list