[llvm] [X86][PartialReduction] Lower zext-byte add reductions to vpsadbw (PR #201076)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Aug 6 05:19:32 PDT 2026
https://github.com/mbhade-amd updated https://github.com/llvm/llvm-project/pull/201076
>From d1f6a75c9aa52674c4127febc6b38b145a00e78e Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Sun, 7 Jun 2026 13:06:26 +0530
Subject: [PATCH 1/3] [X86] Add CodeGen tests for byte-sum reduction patterns
(NFC)
Add byte-sum.ll exercising zext-byte add-reduction shapes that could
benefit from PSADBW lowering, plus negative cases (sext, i16 source,
i16 accumulator, non-zext leaf, VF < 16).
Tests use @llvm.vector.reduce.add.* intrinsics and cover SSE2, AVX2,
and AVX-512BW subtargets. CHECK lines reflect current trunk codegen
(vpmovzxbd + vpaddd widening); a follow-up commit will add the
matcher and update the CHECK lines.
---
llvm/test/CodeGen/X86/byte-sum.ll | 1181 +++++++++++++++++++++++++++++
1 file changed, 1181 insertions(+)
create mode 100644 llvm/test/CodeGen/X86/byte-sum.ll
diff --git a/llvm/test/CodeGen/X86/byte-sum.ll b/llvm/test/CodeGen/X86/byte-sum.ll
new file mode 100644
index 0000000000000..f37ebb0a4cb3f
--- /dev/null
+++ b/llvm/test/CodeGen/X86/byte-sum.ll
@@ -0,0 +1,1181 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX512BW
+
+; CodeGen tests for X86PartialReduction::tryByteSumReplacement.
+; Positive cases should lower to vpsadbw; negative cases should not.
+
+ at a = global [1024 x i8] zeroinitializer, align 16
+
+;============================================================================
+; POSITIVE: zext <16 x i8> to <16 x i32>, i32 accumulator
+;============================================================================
+define i32 @byte_sum_v16_i32() nounwind {
+; SSE2-LABEL: byte_sum_v16_i32:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB0_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm5
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; SSE2-NEXT: movdqa %xmm6, %xmm7
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; SSE2-NEXT: paddd %xmm7, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
+; SSE2-NEXT: paddd %xmm6, %xmm0
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
+; SSE2-NEXT: movdqa %xmm5, %xmm6
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; SSE2-NEXT: paddd %xmm6, %xmm4
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; SSE2-NEXT: paddd %xmm5, %xmm3
+; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: jne .LBB0_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm4, %xmm1
+; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v16_i32:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB0_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: jne .LBB0_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v16_i32:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB0_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: jne .LBB0_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <16 x i8>, ptr %p, align 16
+ %z = zext <16 x i8> %wide.load to <16 x i32>
+ %add = add nsw <16 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
+ ret i32 %ext
+}
+
+;============================================================================
+; POSITIVE: zext <32 x i8> to <32 x i32>, i32 accumulator
+;============================================================================
+define i32 @byte_sum_v32_i32() nounwind {
+; SSE2-LABEL: byte_sum_v32_i32:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm7, %xmm7
+; SSE2-NEXT: pxor %xmm6, %xmm6
+; SSE2-NEXT: pxor %xmm8, %xmm8
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB1_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm9
+; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm10
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm4[8],xmm11[9],xmm4[9],xmm11[10],xmm4[10],xmm11[11],xmm4[11],xmm11[12],xmm4[12],xmm11[13],xmm4[13],xmm11[14],xmm4[14],xmm11[15],xmm4[15]
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm4[4],xmm12[5],xmm4[5],xmm12[6],xmm4[6],xmm12[7],xmm4[7]
+; SSE2-NEXT: paddd %xmm12, %xmm8
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
+; SSE2-NEXT: paddd %xmm11, %xmm6
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]
+; SSE2-NEXT: paddd %xmm11, %xmm7
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3]
+; SSE2-NEXT: paddd %xmm10, %xmm5
+; SSE2-NEXT: movdqa %xmm9, %xmm10
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
+; SSE2-NEXT: paddd %xmm11, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
+; SSE2-NEXT: paddd %xmm10, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15]
+; SSE2-NEXT: movdqa %xmm9, %xmm10
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3]
+; SSE2-NEXT: paddd %xmm10, %xmm0
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm4[4],xmm9[5],xmm4[5],xmm9[6],xmm4[6],xmm9[7],xmm4[7]
+; SSE2-NEXT: paddd %xmm9, %xmm2
+; SSE2-NEXT: addq $32, %rax
+; SSE2-NEXT: jne .LBB1_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm7, %xmm3
+; SSE2-NEXT: paddd %xmm8, %xmm2
+; SSE2-NEXT: paddd %xmm3, %xmm2
+; SSE2-NEXT: paddd %xmm5, %xmm1
+; SSE2-NEXT: paddd %xmm6, %xmm0
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v32_i32:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB1_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm2, %ymm4, %ymm2
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm1, %ymm4, %ymm1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm0, %ymm4, %ymm0
+; AVX2-NEXT: addq $32, %rax
+; AVX2-NEXT: jne .LBB1_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpaddd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v32_i32:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB1_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT: addq $32, %rax
+; AVX512BW-NEXT: jne .LBB1_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <32 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <32 x i8>, ptr %p, align 16
+ %z = zext <32 x i8> %wide.load to <32 x i32>
+ %add = add nsw <32 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 32
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v32i32(<32 x i32> %add)
+ ret i32 %ext
+}
+
+;============================================================================
+; POSITIVE: zext <64 x i8> to <64 x i32>, i32 accumulator
+;============================================================================
+define i32 @byte_sum_v64_i32() nounwind {
+; SSE2-LABEL: byte_sum_v64_i32:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm10, %xmm10
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm9, %xmm9
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm13, %xmm13
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: pxor %xmm12, %xmm12
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm15, %xmm15
+; SSE2-NEXT: pxor %xmm7, %xmm7
+; SSE2-NEXT: pxor %xmm6, %xmm6
+; SSE2-NEXT: pxor %xmm11, %xmm11
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm8, %xmm8
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: pxor %xmm14, %xmm14
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB2_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa 1072(%rcx,%rax), %xmm3
+; SSE2-NEXT: movdqa %xmm3, %xmm0
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm10[8],xmm0[9],xmm10[9],xmm0[10],xmm10[10],xmm0[11],xmm10[11],xmm0[12],xmm10[12],xmm0[13],xmm10[13],xmm0[14],xmm10[14],xmm0[15],xmm10[15]
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
+; SSE2-NEXT: paddd %xmm1, %xmm14
+; SSE2-NEXT: movdqa 1056(%rcx,%rax), %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; SSE2-NEXT: paddd %xmm0, %xmm4
+; SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3],xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
+; SSE2-NEXT: movdqa %xmm3, %xmm0
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT: paddd %xmm0, %xmm8
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm3, %xmm2
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm10[8],xmm3[9],xmm10[9],xmm3[10],xmm10[10],xmm3[11],xmm10[11],xmm3[12],xmm10[12],xmm3[13],xmm10[13],xmm3[14],xmm10[14],xmm3[15],xmm10[15]
+; SSE2-NEXT: movdqa %xmm3, %xmm0
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT: paddd %xmm0, %xmm11
+; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm0
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm3, %xmm6
+; SSE2-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3],xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
+; SSE2-NEXT: movdqa %xmm1, %xmm3
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
+; SSE2-NEXT: paddd %xmm3, %xmm7
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm1, %xmm15
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm10[8],xmm3[9],xmm10[9],xmm3[10],xmm10[10],xmm3[11],xmm10[11],xmm3[12],xmm10[12],xmm3[13],xmm10[13],xmm3[14],xmm10[14],xmm3[15],xmm10[15]
+; SSE2-NEXT: movdqa %xmm3, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
+; SSE2-NEXT: paddd %xmm1, %xmm5
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm1
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm3, %xmm12
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
+; SSE2-NEXT: movdqa %xmm2, %xmm6
+; SSE2-NEXT: movdqa %xmm15, %xmm2
+; SSE2-NEXT: movdqa %xmm14, %xmm15
+; SSE2-NEXT: movdqa %xmm13, %xmm14
+; SSE2-NEXT: movdqa %xmm11, %xmm13
+; SSE2-NEXT: movdqa %xmm12, %xmm11
+; SSE2-NEXT: movdqa %xmm8, %xmm12
+; SSE2-NEXT: movdqa %xmm9, %xmm8
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
+; SSE2-NEXT: paddd %xmm3, %xmm4
+; SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; SSE2-NEXT: movdqa %xmm8, %xmm9
+; SSE2-NEXT: movdqa %xmm12, %xmm8
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: movdqa %xmm13, %xmm11
+; SSE2-NEXT: movdqa %xmm14, %xmm13
+; SSE2-NEXT: movdqa %xmm15, %xmm14
+; SSE2-NEXT: movdqa %xmm2, %xmm15
+; SSE2-NEXT: movdqa %xmm6, %xmm2
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm0, %xmm3
+; SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT: movdqa %xmm0, %xmm3
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm3, %xmm9
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
+; SSE2-NEXT: paddd %xmm0, %xmm3
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm10[8],xmm1[9],xmm10[9],xmm1[10],xmm10[10],xmm1[11],xmm10[11],xmm1[12],xmm10[12],xmm1[13],xmm10[13],xmm1[14],xmm10[14],xmm1[15],xmm10[15]
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
+; SSE2-NEXT: paddd %xmm0, %xmm13
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT: addq $64, %rax
+; SSE2-NEXT: jne .LBB2_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm6, %xmm13
+; SSE2-NEXT: paddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
+; SSE2-NEXT: paddd %xmm13, %xmm12
+; SSE2-NEXT: paddd %xmm15, %xmm9
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
+; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: paddd %xmm9, %xmm0
+; SSE2-NEXT: paddd %xmm12, %xmm0
+; SSE2-NEXT: paddd %xmm11, %xmm1
+; SSE2-NEXT: paddd %xmm14, %xmm5
+; SSE2-NEXT: paddd %xmm1, %xmm5
+; SSE2-NEXT: paddd %xmm7, %xmm3
+; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT: paddd %xmm8, %xmm1
+; SSE2-NEXT: paddd %xmm3, %xmm1
+; SSE2-NEXT: paddd %xmm5, %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v64_i32:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: vpxor %xmm6, %xmm6, %xmm6
+; AVX2-NEXT: vpxor %xmm5, %xmm5, %xmm5
+; AVX2-NEXT: vpxor %xmm7, %xmm7, %xmm7
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB2_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm7, %ymm8, %ymm7
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm5, %ymm8, %ymm5
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm6, %ymm8, %ymm6
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm3, %ymm8, %ymm3
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm4, %ymm8, %ymm4
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm2, %ymm8, %ymm2
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm1, %ymm8, %ymm1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm0, %ymm8, %ymm0
+; AVX2-NEXT: addq $64, %rax
+; AVX2-NEXT: jne .LBB2_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddd %ymm6, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm7, %ymm4, %ymm4
+; AVX2-NEXT: vpaddd %ymm4, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm3, %ymm1, %ymm1
+; AVX2-NEXT: vpaddd %ymm5, %ymm2, %ymm2
+; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v64_i32:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX512BW-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB2_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm3, %zmm4, %zmm3
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm2, %zmm4, %zmm2
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm4, %zmm1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm4, %zmm0
+; AVX512BW-NEXT: addq $64, %rax
+; AVX512BW-NEXT: jne .LBB2_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vpaddd %zmm2, %zmm1, %zmm1
+; AVX512BW-NEXT: vpaddd %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <64 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <64 x i8>, ptr %p, align 16
+ %z = zext <64 x i8> %wide.load to <64 x i32>
+ %add = add nsw <64 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 64
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v64i32(<64 x i32> %add)
+ ret i32 %ext
+}
+
+;============================================================================
+; POSITIVE: zext <16 x i8> to <16 x i64>, i64 accumulator
+;============================================================================
+define i64 @byte_sum_v16_i64() nounwind {
+; SSE2-LABEL: byte_sum_v16_i64:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: pxor %xmm7, %xmm7
+; SSE2-NEXT: pxor %xmm6, %xmm6
+; SSE2-NEXT: pxor %xmm8, %xmm8
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB3_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm10
+; SSE2-NEXT: movdqa %xmm10, %xmm9
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15]
+; SSE2-NEXT: movdqa %xmm9, %xmm11
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm12 = xmm12[2],xmm4[2],xmm12[3],xmm4[3]
+; SSE2-NEXT: paddq %xmm12, %xmm8
+; SSE2-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1]
+; SSE2-NEXT: paddq %xmm11, %xmm6
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
+; SSE2-NEXT: movdqa %xmm9, %xmm11
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm4[2],xmm11[3],xmm4[3]
+; SSE2-NEXT: paddq %xmm11, %xmm7
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
+; SSE2-NEXT: movdqa %xmm11, %xmm12
+; SSE2-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm4[0],xmm12[1],xmm4[1]
+; SSE2-NEXT: paddq %xmm12, %xmm1
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm4[2],xmm11[3],xmm4[3]
+; SSE2-NEXT: paddq %xmm11, %xmm3
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
+; SSE2-NEXT: movdqa %xmm10, %xmm11
+; SSE2-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1]
+; SSE2-NEXT: paddq %xmm11, %xmm0
+; SSE2-NEXT: punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm4[2],xmm10[3],xmm4[3]
+; SSE2-NEXT: paddq %xmm10, %xmm2
+; SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
+; SSE2-NEXT: paddq %xmm9, %xmm5
+; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: jne .LBB3_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddq %xmm7, %xmm3
+; SSE2-NEXT: paddq %xmm8, %xmm2
+; SSE2-NEXT: paddq %xmm3, %xmm2
+; SSE2-NEXT: paddq %xmm5, %xmm1
+; SSE2-NEXT: paddq %xmm6, %xmm0
+; SSE2-NEXT: paddq %xmm1, %xmm0
+; SSE2-NEXT: paddq %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddq %xmm0, %xmm1
+; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v16_i64:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
+; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB3_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpaddq %ymm3, %ymm4, %ymm3
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpaddq %ymm2, %ymm4, %ymm2
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpaddq %ymm1, %ymm4, %ymm1
+; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
+; AVX2-NEXT: vpaddq %ymm0, %ymm4, %ymm0
+; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: jne .LBB3_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddq %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpaddq %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovq %xmm0, %rax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v16_i64:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB3_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxbq {{.*#+}} zmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT: vpaddq %zmm1, %zmm2, %zmm1
+; AVX512BW-NEXT: vpmovzxbq {{.*#+}} zmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
+; AVX512BW-NEXT: vpaddq %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: jne .LBB3_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vpaddq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddq %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddq %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovq %xmm0, %rax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <16 x i8>, ptr %p, align 16
+ %z = zext <16 x i8> %wide.load to <16 x i64>
+ %add = add nsw <16 x i64> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %add)
+ ret i64 %ext
+}
+
+;============================================================================
+; NEGATIVE: sext instead of zext
+;============================================================================
+define i32 @byte_sum_v16_i32_sext() nounwind {
+; SSE2-LABEL: byte_sum_v16_i32_sext:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB4_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
+; SSE2-NEXT: psrad $24, %xmm6
+; SSE2-NEXT: paddd %xmm6, %xmm0
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4,4,5,5,6,6,7,7]
+; SSE2-NEXT: psrad $24, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm1
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm4 = xmm4[8,8,9,9,10,10,11,11,12,12,13,13,14,14,15,15]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3]
+; SSE2-NEXT: psrad $24, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm3
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm4 = xmm4[4,4,5,5,6,6,7,7]
+; SSE2-NEXT: psrad $24, %xmm4
+; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: jne .LBB4_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm2, %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v16_i32_sext:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB4_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovsxbd 1024(%rcx,%rax), %ymm2
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpmovsxbd 1032(%rcx,%rax), %ymm2
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: jne .LBB4_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v16_i32_sext:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB4_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovsxbd 1024(%rcx,%rax), %zmm1
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: jne .LBB4_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <16 x i8>, ptr %p, align 16
+ %z = sext <16 x i8> %wide.load to <16 x i32>
+ %add = add nsw <16 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
+ ret i32 %ext
+}
+
+;============================================================================
+; NEGATIVE: i16 source instead of i8
+;============================================================================
+define i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
+; SSE2-LABEL: word_sum_v16_i32:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: movq $-2048, %rax # imm = 0xF800
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB5_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqu 2048(%rdi,%rax), %xmm5
+; SSE2-NEXT: movdqu 2064(%rdi,%rax), %xmm6
+; SSE2-NEXT: movdqa %xmm6, %xmm7
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
+; SSE2-NEXT: paddd %xmm7, %xmm3
+; SSE2-NEXT: movdqa %xmm5, %xmm7
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
+; SSE2-NEXT: paddd %xmm7, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
+; SSE2-NEXT: paddd %xmm5, %xmm0
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; SSE2-NEXT: paddd %xmm6, %xmm4
+; SSE2-NEXT: addq $32, %rax
+; SSE2-NEXT: jne .LBB5_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm4, %xmm1
+; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: word_sum_v16_i32:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-2048, %rax # imm = 0xF800
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB5_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: addq $32, %rax
+; AVX2-NEXT: jne .LBB5_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: word_sum_v16_i32:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-2048, %rax # imm = 0xF800
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB5_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: addq $32, %rax
+; AVX512BW-NEXT: jne .LBB5_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds i16, ptr %src, i64 %index
+ %wide.load = load <16 x i16>, ptr %p, align 2
+ %z = zext <16 x i16> %wide.load to <16 x i32>
+ %add = add nsw <16 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
+ ret i32 %ext
+}
+
+;============================================================================
+; NEGATIVE: i16 accumulator (element type < 32 bits)
+;============================================================================
+define i16 @byte_sum_v16_i16() nounwind {
+; SSE2-LABEL: byte_sum_v16_i16:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB6_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm3
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
+; SSE2-NEXT: paddw %xmm4, %xmm0
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; SSE2-NEXT: paddw %xmm3, %xmm2
+; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: jne .LBB6_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddw %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddw %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddw %xmm1, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm1
+; SSE2-NEXT: psrld $16, %xmm1
+; SSE2-NEXT: paddw %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v16_i16:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB6_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; AVX2-NEXT: vpaddw %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: jne .LBB6_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX2-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v16_i16:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB6_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
+; AVX512BW-NEXT: vpaddw %ymm0, %ymm1, %ymm0
+; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: jne .LBB6_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpsrld $16, %xmm0, %xmm1
+; AVX512BW-NEXT: vpaddw %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: # kill: def $ax killed $ax killed $eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i16> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <16 x i8>, ptr %p, align 16
+ %z = zext <16 x i8> %wide.load to <16 x i16>
+ %add = add nsw <16 x i16> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %add)
+ ret i16 %ext
+}
+
+;============================================================================
+; NEGATIVE: leaf is not a zext (already i32 values)
+;============================================================================
+define i32 @dword_sum_v16_i32(ptr nocapture readonly %src) nounwind {
+; SSE2-LABEL: dword_sum_v16_i32:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: movq $-4096, %rax # imm = 0xF000
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: pxor %xmm3, %xmm3
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB7_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movdqu 4096(%rdi,%rax), %xmm4
+; SSE2-NEXT: paddd %xmm4, %xmm0
+; SSE2-NEXT: movdqu 4112(%rdi,%rax), %xmm4
+; SSE2-NEXT: paddd %xmm4, %xmm1
+; SSE2-NEXT: movdqu 4128(%rdi,%rax), %xmm4
+; SSE2-NEXT: paddd %xmm4, %xmm3
+; SSE2-NEXT: movdqu 4144(%rdi,%rax), %xmm4
+; SSE2-NEXT: paddd %xmm4, %xmm2
+; SSE2-NEXT: addq $64, %rax
+; SSE2-NEXT: jne .LBB7_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm2, %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: dword_sum_v16_i32:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-4096, %rax # imm = 0xF000
+; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB7_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpaddd 4096(%rdi,%rax), %ymm1, %ymm1
+; AVX2-NEXT: vpaddd 4128(%rdi,%rax), %ymm0, %ymm0
+; AVX2-NEXT: addq $64, %rax
+; AVX2-NEXT: jne .LBB7_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: dword_sum_v16_i32:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-4096, %rax # imm = 0xF000
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB7_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpaddd 4096(%rdi,%rax), %zmm0, %zmm0
+; AVX512BW-NEXT: addq $64, %rax
+; AVX512BW-NEXT: jne .LBB7_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds i32, ptr %src, i64 %index
+ %wide.load = load <16 x i32>, ptr %p, align 4
+ %add = add nsw <16 x i32> %wide.load, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
+ ret i32 %ext
+}
+
+;============================================================================
+; NEGATIVE: VF = 8 (fewer than 16 lanes)
+;============================================================================
+define i32 @byte_sum_v8_i32() nounwind {
+; SSE2-LABEL: byte_sum_v8_i32:
+; SSE2: # %bb.0: # %entry
+; SSE2-NEXT: pxor %xmm1, %xmm1
+; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: .p2align 4
+; SSE2-NEXT: .LBB8_1: # %vector.body
+; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE2-NEXT: movq {{.*#+}} xmm3 = mem[0],zero
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT: movdqa %xmm3, %xmm4
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
+; SSE2-NEXT: paddd %xmm4, %xmm0
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT: paddd %xmm3, %xmm2
+; SSE2-NEXT: addq $8, %rax
+; SSE2-NEXT: jne .LBB8_1
+; SSE2-NEXT: # %bb.2: # %middle.block
+; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: retq
+;
+; AVX2-LABEL: byte_sum_v8_i32:
+; AVX2: # %bb.0: # %entry
+; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX2-NEXT: .p2align 4
+; AVX2-NEXT: .LBB8_1: # %vector.body
+; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: addq $8, %rax
+; AVX2-NEXT: jne .LBB8_1
+; AVX2-NEXT: # %bb.2: # %middle.block
+; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX2-NEXT: vmovd %xmm0, %eax
+; AVX2-NEXT: vzeroupper
+; AVX2-NEXT: retq
+;
+; AVX512BW-LABEL: byte_sum_v8_i32:
+; AVX512BW: # %bb.0: # %entry
+; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
+; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: .p2align 4
+; AVX512BW-NEXT: .LBB8_1: # %vector.body
+; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
+; AVX512BW-NEXT: vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
+; AVX512BW-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX512BW-NEXT: addq $8, %rax
+; AVX512BW-NEXT: jne .LBB8_1
+; AVX512BW-NEXT: # %bb.2: # %middle.block
+; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; AVX512BW-NEXT: vpaddd %xmm1, %xmm0, %xmm0
+; AVX512BW-NEXT: vmovd %xmm0, %eax
+; AVX512BW-NEXT: vzeroupper
+; AVX512BW-NEXT: retq
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <8 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <8 x i8>, ptr %p, align 8
+ %z = zext <8 x i8> %wide.load to <8 x i32>
+ %add = add nsw <8 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 8
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %add)
+ ret i32 %ext
+}
>From 2895c930fa867542a6d75b4dcc357a84d21af5e4 Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Sun, 7 Jun 2026 13:20:57 +0530
Subject: [PATCH 2/3] [X86][PartialReduction] Lower zext-byte add reductions to
vpsadbw
Loops of the form `for (i) sum += bytes[i];` (uint8_t input, i32 or
i64 accumulator) currently lower to vpmovzxbd + vpaddd even though
PSADBW(x, 0) computes the same sum natively in one instruction per
128/256/512-bit lane.
Teach X86PartialReduction's matcher to rewrite the
`zext <N x i8> to <N x i32|i64>` leaves of an add reduction
(N >= 16) into PSADBW(x, 0). The rewrite splits the source into as
many SSE2/AVX2/AVX-512BW lanes as the subtarget and VF require;
i64 accumulators consume PSADBW's natural <N/8 x i64> output
directly without an intermediate i32 bitcast.
x86-partial-reduction-byte-sum*.ll isolate the matcher with
`opt -passes=x86-partial-reduction` (positive / negative / wide-i64
/ debug-location preservation). byte-sum.ll exercises the full
CodeGen pipeline with `llc` on +sse2, +avx2 and +avx512bw.
---
llvm/lib/Target/X86/X86PartialReduction.cpp | 116 ++++-
llvm/test/CodeGen/X86/byte-sum.ll | 400 ++++--------------
...x86-partial-reduction-byte-sum-debugloc.ll | 43 ++
...x86-partial-reduction-byte-sum-negative.ll | 49 +++
...x86-partial-reduction-byte-sum-wide-i64.ll | 59 +++
.../X86/x86-partial-reduction-byte-sum.ll | 26 ++
6 files changed, 386 insertions(+), 307 deletions(-)
create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll
create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll
create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll
create mode 100644 llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
diff --git a/llvm/lib/Target/X86/X86PartialReduction.cpp b/llvm/lib/Target/X86/X86PartialReduction.cpp
index a017ed9f2ff50..85092ae569afd 100644
--- a/llvm/lib/Target/X86/X86PartialReduction.cpp
+++ b/llvm/lib/Target/X86/X86PartialReduction.cpp
@@ -18,6 +18,7 @@
#include "llvm/CodeGen/TargetPassConfig.h"
#include "llvm/IR/Analysis.h"
#include "llvm/IR/Constants.h"
+#include "llvm/IR/DerivedTypes.h"
#include "llvm/IR/IRBuilder.h"
#include "llvm/IR/Instructions.h"
#include "llvm/IR/IntrinsicsX86.h"
@@ -44,6 +45,7 @@ class X86PartialReduction {
private:
bool tryMAddReplacement(Instruction *Op, bool ReduceInOneBB);
bool trySADReplacement(Instruction *Op);
+ bool tryByteSumReplacement(Instruction *Op);
};
class X86PartialReductionLegacy : public FunctionPass {
@@ -353,6 +355,106 @@ bool X86PartialReduction::trySADReplacement(Instruction *Op) {
return true;
}
+bool X86PartialReduction::tryByteSumReplacement(Instruction *Op) {
+ if (!ST->hasSSE2())
+ return false;
+
+ auto *OpTy = dyn_cast<FixedVectorType>(Op->getType());
+ if (!OpTy)
+ return false;
+ unsigned ElemBits = OpTy->getElementType()->getScalarSizeInBits();
+ if (ElemBits != 32 && ElemBits != 64)
+ return false;
+
+ auto *ZExt = dyn_cast<ZExtInst>(Op);
+ if (!ZExt)
+ return false;
+
+ auto *SrcTy = dyn_cast<FixedVectorType>(ZExt->getOperand(0)->getType());
+ if (!SrcTy || !SrcTy->getElementType()->isIntegerTy(8))
+ return false;
+
+ unsigned NumElts = OpTy->getNumElements();
+
+ // Below 16 elements, SelectionDAG's SAD matcher handles it.
+ if (NumElts < 16)
+ return false;
+
+ // Select the widest psadbw intrinsic the subtarget supports.
+ unsigned IntrinsicNumElts;
+ Intrinsic::ID IID;
+ if (ST->useBWIRegs() && NumElts >= 64) {
+ IID = Intrinsic::x86_avx512_psad_bw_512;
+ IntrinsicNumElts = 64;
+ } else if (ST->hasAVX2() && NumElts >= 32) {
+ IID = Intrinsic::x86_avx2_psad_bw;
+ IntrinsicNumElts = 32;
+ } else {
+ IID = Intrinsic::x86_sse2_psad_bw;
+ IntrinsicNumElts = 16;
+ }
+
+ if (NumElts % IntrinsicNumElts != 0 ||
+ !isPowerOf2_32(NumElts / IntrinsicNumElts))
+ return false;
+ unsigned NumSplits = NumElts / IntrinsicNumElts;
+
+ IRBuilder<> Builder(Op);
+ Builder.SetCurrentDebugLocation(Op->getDebugLoc());
+
+ Function *PSADBWFn = Intrinsic::getOrInsertDeclaration(Op->getModule(), IID);
+
+ // psadbw(x, 0) horizontally sums 8 bytes per lane into i64.
+ auto *I8VecTy = FixedVectorType::get(Builder.getInt8Ty(), IntrinsicNumElts);
+ Value *Zeroes = Constant::getNullValue(I8VecTy);
+
+ // For i32 accumulators, bitcast each i64 lane to two i32 lanes.
+ // Per-lane sums are at most 8*255 = 2040, so the upper i32 is always zero.
+ FixedVectorType *I32PerSplitTy =
+ ElemBits == 32
+ ? FixedVectorType::get(Builder.getInt32Ty(), IntrinsicNumElts / 4)
+ : nullptr;
+
+ // Split input into IntrinsicNumElts-byte lanes and compute psadbw per lane.
+ Value *Src = ZExt->getOperand(0);
+ SmallVector<Value *, 4> Ops(NumSplits);
+ for (unsigned i = 0; i != NumSplits; ++i) {
+ SmallVector<int, 64> ExtractMask(IntrinsicNumElts);
+ std::iota(ExtractMask.begin(), ExtractMask.end(), i * IntrinsicNumElts);
+ Value *ExtractSrc = Builder.CreateShuffleVector(Src, Src, ExtractMask);
+ Ops[i] = Builder.CreateCall(PSADBWFn, {ExtractSrc, Zeroes});
+ if (I32PerSplitTy)
+ Ops[i] = Builder.CreateBitCast(Ops[i], I32PerSplitTy);
+ }
+
+ // Concat per-split results with a pairwise shuffle tree.
+ unsigned Stages = Log2_32(NumSplits);
+ for (unsigned S = Stages; S > 0; --S) {
+ unsigned NumConcatElts =
+ cast<FixedVectorType>(Ops[0]->getType())->getNumElements() * 2;
+ for (unsigned i = 0; i != 1U << (S - 1); ++i) {
+ SmallVector<int, 64> ConcatMask(NumConcatElts);
+ std::iota(ConcatMask.begin(), ConcatMask.end(), 0);
+ Ops[i] =
+ Builder.CreateShuffleVector(Ops[i * 2], Ops[i * 2 + 1], ConcatMask);
+ }
+ }
+
+ // Pad with zeros to match the original vector width.
+ SmallVector<int, 32> ConcatMask(NumElts);
+ unsigned SubElts = cast<FixedVectorType>(Ops[0]->getType())->getNumElements();
+ for (unsigned i = 0; i != SubElts; ++i)
+ ConcatMask[i] = i;
+ for (unsigned i = SubElts; i != NumElts; ++i)
+ ConcatMask[i] = (i % SubElts) + SubElts;
+ Value *Zero = Constant::getNullValue(Ops[0]->getType());
+ Ops[0] = Builder.CreateShuffleVector(Ops[0], Zero, ConcatMask);
+
+ Op->replaceAllUsesWith(Ops[0]);
+ Op->eraseFromParent();
+ return true;
+}
+
// Walk backwards from the ExtractElementInst and determine if it is the end of
// a horizontal reduction. Return the input to the reduction if we find one.
static Value *matchAddReduction(const ExtractElementInst &EE,
@@ -530,8 +632,20 @@ bool X86PartialReduction::run(Function &F) {
// Don't do SAD matching on the root node. SelectionDAG already
// has support for that and currently generates better code.
- if (I != Root && trySADReplacement(I))
+ if (I != Root && trySADReplacement(I)) {
MadeChange = true;
+ continue;
+ }
+
+ // Byte sum via psadbw(x, 0). Same rationale as trySADReplacement:
+ // don't match on the root node because SelectionDAG already handles
+ // small single-vector patterns and generally emits better code for
+ // them. We only help on wider intermediate shapes that reach us
+ // from loop vectorization.
+ if (I != Root && tryByteSumReplacement(I)) {
+ MadeChange = true;
+ continue;
+ }
}
}
}
diff --git a/llvm/test/CodeGen/X86/byte-sum.ll b/llvm/test/CodeGen/X86/byte-sum.ll
index f37ebb0a4cb3f..0a22fe31b46f7 100644
--- a/llvm/test/CodeGen/X86/byte-sum.ll
+++ b/llvm/test/CodeGen/X86/byte-sum.ll
@@ -14,35 +14,21 @@
define i32 @byte_sum_v16_i32() nounwind {
; SSE2-LABEL: byte_sum_v16_i32:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm2, %xmm2
+; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: pxor %xmm4, %xmm4
-; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB0_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm5
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3],xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; SSE2-NEXT: movdqa %xmm6, %xmm7
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm7 = xmm7[0],xmm2[0],xmm7[1],xmm2[1],xmm7[2],xmm2[2],xmm7[3],xmm2[3]
-; SSE2-NEXT: paddd %xmm7, %xmm1
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm6 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; SSE2-NEXT: paddd %xmm6, %xmm0
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm5 = xmm5[8],xmm2[8],xmm5[9],xmm2[9],xmm5[10],xmm2[10],xmm5[11],xmm2[11],xmm5[12],xmm2[12],xmm5[13],xmm2[13],xmm5[14],xmm2[14],xmm5[15],xmm2[15]
-; SSE2-NEXT: movdqa %xmm5, %xmm6
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
-; SSE2-NEXT: paddd %xmm6, %xmm4
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm5 = xmm5[4],xmm2[4],xmm5[5],xmm2[5],xmm5[6],xmm2[6],xmm5[7],xmm2[7]
-; SSE2-NEXT: paddd %xmm5, %xmm3
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm2
+; SSE2-NEXT: psadbw %xmm0, %xmm2
+; SSE2-NEXT: paddd %xmm2, %xmm1
; SSE2-NEXT: addq $16, %rax
; SSE2-NEXT: jne .LBB0_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm4, %xmm1
-; SSE2-NEXT: paddd %xmm3, %xmm0
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm0
; SSE2-NEXT: paddd %xmm1, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddd %xmm0, %xmm1
@@ -57,17 +43,16 @@ define i32 @byte_sum_v16_i32() nounwind {
; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB0_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm3
+; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; AVX2-NEXT: addq $16, %rax
; AVX2-NEXT: jne .LBB0_1
; AVX2-NEXT: # %bb.2: # %middle.block
-; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -83,11 +68,12 @@ define i32 @byte_sum_v16_i32() nounwind {
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
+; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB0_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm2
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
; AVX512BW-NEXT: addq $16, %rax
; AVX512BW-NEXT: jne .LBB0_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
@@ -127,58 +113,29 @@ middle.block:
define i32 @byte_sum_v32_i32() nounwind {
; SSE2-LABEL: byte_sum_v32_i32:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: pxor %xmm7, %xmm7
-; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pxor %xmm8, %xmm8
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB1_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm9
-; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm10
-; SSE2-NEXT: movdqa %xmm10, %xmm11
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm11 = xmm11[8],xmm4[8],xmm11[9],xmm4[9],xmm11[10],xmm4[10],xmm11[11],xmm4[11],xmm11[12],xmm4[12],xmm11[13],xmm4[13],xmm11[14],xmm4[14],xmm11[15],xmm4[15]
-; SSE2-NEXT: movdqa %xmm11, %xmm12
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm12 = xmm12[4],xmm4[4],xmm12[5],xmm4[5],xmm12[6],xmm4[6],xmm12[7],xmm4[7]
-; SSE2-NEXT: paddd %xmm12, %xmm8
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
-; SSE2-NEXT: paddd %xmm11, %xmm6
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; SSE2-NEXT: movdqa %xmm10, %xmm11
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]
-; SSE2-NEXT: paddd %xmm11, %xmm7
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3]
-; SSE2-NEXT: paddd %xmm10, %xmm5
-; SSE2-NEXT: movdqa %xmm9, %xmm10
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; SSE2-NEXT: movdqa %xmm10, %xmm11
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
-; SSE2-NEXT: paddd %xmm11, %xmm1
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; SSE2-NEXT: paddd %xmm10, %xmm3
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15]
-; SSE2-NEXT: movdqa %xmm9, %xmm10
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3]
-; SSE2-NEXT: paddd %xmm10, %xmm0
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm9 = xmm9[4],xmm4[4],xmm9[5],xmm4[5],xmm9[6],xmm4[6],xmm9[7],xmm4[7]
-; SSE2-NEXT: paddd %xmm9, %xmm2
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm3
+; SSE2-NEXT: psadbw %xmm0, %xmm3
+; SSE2-NEXT: paddd %xmm3, %xmm1
+; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm3
+; SSE2-NEXT: psadbw %xmm0, %xmm3
+; SSE2-NEXT: paddd %xmm3, %xmm2
; SSE2-NEXT: addq $32, %rax
; SSE2-NEXT: jne .LBB1_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm7, %xmm3
-; SSE2-NEXT: paddd %xmm8, %xmm2
-; SSE2-NEXT: paddd %xmm3, %xmm2
-; SSE2-NEXT: paddd %xmm5, %xmm1
-; SSE2-NEXT: paddd %xmm6, %xmm0
-; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: paddd %xmm0, %xmm2
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: paddd %xmm0, %xmm0
+; SSE2-NEXT: paddd %xmm0, %xmm1
; SSE2-NEXT: paddd %xmm2, %xmm0
+; SSE2-NEXT: paddd %xmm1, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddd %xmm0, %xmm1
; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
@@ -192,24 +149,16 @@ define i32 @byte_sum_v32_i32() nounwind {
; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB1_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm2, %ymm4, %ymm2
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm1, %ymm4, %ymm1
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm0, %ymm4, %ymm0
+; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %ymm0, %ymm2
+; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
; AVX2-NEXT: addq $32, %rax
; AVX2-NEXT: jne .LBB1_1
; AVX2-NEXT: # %bb.2: # %middle.block
-; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpaddd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm1
+; AVX2-NEXT: vpaddd %ymm0, %ymm0, %ymm0
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
@@ -227,17 +176,16 @@ define i32 @byte_sum_v32_i32() nounwind {
; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB1_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %ymm1, %ymm3
+; AVX512BW-NEXT: vpaddd %zmm2, %zmm3, %zmm2
; AVX512BW-NEXT: addq $32, %rax
; AVX512BW-NEXT: jne .LBB1_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
-; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
@@ -274,137 +222,46 @@ middle.block:
define i32 @byte_sum_v64_i32() nounwind {
; SSE2-LABEL: byte_sum_v64_i32:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm10, %xmm10
+; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
-; SSE2-NEXT: pxor %xmm9, %xmm9
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pxor %xmm13, %xmm13
-; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: pxor %xmm12, %xmm12
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: pxor %xmm15, %xmm15
-; SSE2-NEXT: pxor %xmm7, %xmm7
-; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pxor %xmm11, %xmm11
+; SSE2-NEXT: pxor %xmm4, %xmm4
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm8, %xmm8
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: pxor %xmm14, %xmm14
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB2_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa %xmm1, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa 1072(%rcx,%rax), %xmm3
-; SSE2-NEXT: movdqa %xmm3, %xmm0
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm0 = xmm0[8],xmm10[8],xmm0[9],xmm10[9],xmm0[10],xmm10[10],xmm0[11],xmm10[11],xmm0[12],xmm10[12],xmm0[13],xmm10[13],xmm0[14],xmm10[14],xmm0[15],xmm10[15]
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm1, %xmm14
-; SSE2-NEXT: movdqa 1056(%rcx,%rax), %xmm1
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; SSE2-NEXT: paddd %xmm0, %xmm4
-; SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3],xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
-; SSE2-NEXT: movdqa %xmm3, %xmm0
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm0, %xmm8
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm3, %xmm2
-; SSE2-NEXT: movdqa %xmm1, %xmm3
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm10[8],xmm3[9],xmm10[9],xmm3[10],xmm10[10],xmm3[11],xmm10[11],xmm3[12],xmm10[12],xmm3[13],xmm10[13],xmm3[14],xmm10[14],xmm3[15],xmm10[15]
-; SSE2-NEXT: movdqa %xmm3, %xmm0
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm0, %xmm11
-; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm3, %xmm6
-; SSE2-NEXT: movdqa %xmm6, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3],xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
-; SSE2-NEXT: movdqa %xmm1, %xmm3
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm3, %xmm7
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm1 = xmm1[0],xmm10[0],xmm1[1],xmm10[1],xmm1[2],xmm10[2],xmm1[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm1, %xmm15
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm10[8],xmm3[9],xmm10[9],xmm3[10],xmm10[10],xmm3[11],xmm10[11],xmm3[12],xmm10[12],xmm3[13],xmm10[13],xmm3[14],xmm10[14],xmm3[15],xmm10[15]
-; SSE2-NEXT: movdqa %xmm3, %xmm1
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm1, %xmm5
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm1
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm3, %xmm12
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
-; SSE2-NEXT: movdqa %xmm2, %xmm6
-; SSE2-NEXT: movdqa %xmm15, %xmm2
-; SSE2-NEXT: movdqa %xmm14, %xmm15
-; SSE2-NEXT: movdqa %xmm13, %xmm14
-; SSE2-NEXT: movdqa %xmm11, %xmm13
-; SSE2-NEXT: movdqa %xmm12, %xmm11
-; SSE2-NEXT: movdqa %xmm8, %xmm12
-; SSE2-NEXT: movdqa %xmm9, %xmm8
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm4 # 16-byte Reload
-; SSE2-NEXT: paddd %xmm3, %xmm4
-; SSE2-NEXT: movdqa %xmm4, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; SSE2-NEXT: movdqa %xmm8, %xmm9
-; SSE2-NEXT: movdqa %xmm12, %xmm8
-; SSE2-NEXT: movdqa %xmm11, %xmm12
-; SSE2-NEXT: movdqa %xmm13, %xmm11
-; SSE2-NEXT: movdqa %xmm14, %xmm13
-; SSE2-NEXT: movdqa %xmm15, %xmm14
-; SSE2-NEXT: movdqa %xmm2, %xmm15
-; SSE2-NEXT: movdqa %xmm6, %xmm2
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm6 # 16-byte Reload
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm0, %xmm3
-; SSE2-NEXT: movdqa %xmm3, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3],xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
-; SSE2-NEXT: movdqa %xmm0, %xmm3
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm3 = xmm3[0],xmm10[0],xmm3[1],xmm10[1],xmm3[2],xmm10[2],xmm3[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm3, %xmm9
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm3 # 16-byte Reload
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm0 = xmm0[4],xmm10[4],xmm0[5],xmm10[5],xmm0[6],xmm10[6],xmm0[7],xmm10[7]
-; SSE2-NEXT: paddd %xmm0, %xmm3
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm1 = xmm1[8],xmm10[8],xmm1[9],xmm10[9],xmm1[10],xmm10[10],xmm1[11],xmm10[11],xmm1[12],xmm10[12],xmm1[13],xmm10[13],xmm1[14],xmm10[14],xmm1[15],xmm10[15]
-; SSE2-NEXT: movdqa %xmm1, %xmm0
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm0 = xmm0[0],xmm10[0],xmm0[1],xmm10[1],xmm0[2],xmm10[2],xmm0[3],xmm10[3]
-; SSE2-NEXT: paddd %xmm0, %xmm13
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm1 = xmm1[4],xmm10[4],xmm1[5],xmm10[5],xmm1[6],xmm10[6],xmm1[7],xmm10[7]
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: paddd %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 16-byte Spill
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm5
+; SSE2-NEXT: psadbw %xmm3, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm0
+; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm5
+; SSE2-NEXT: psadbw %xmm3, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm4
+; SSE2-NEXT: movdqa 1056(%rcx,%rax), %xmm5
+; SSE2-NEXT: psadbw %xmm3, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm2
+; SSE2-NEXT: movdqa 1072(%rcx,%rax), %xmm5
+; SSE2-NEXT: psadbw %xmm3, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm1
; SSE2-NEXT: addq $64, %rax
; SSE2-NEXT: jne .LBB2_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm6, %xmm13
-; SSE2-NEXT: paddd {{[-0-9]+}}(%r{{[sb]}}p), %xmm12 # 16-byte Folded Reload
-; SSE2-NEXT: paddd %xmm13, %xmm12
-; SSE2-NEXT: paddd %xmm15, %xmm9
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm0 # 16-byte Reload
-; SSE2-NEXT: paddd %xmm2, %xmm0
-; SSE2-NEXT: paddd %xmm9, %xmm0
-; SSE2-NEXT: paddd %xmm12, %xmm0
-; SSE2-NEXT: paddd %xmm11, %xmm1
-; SSE2-NEXT: paddd %xmm14, %xmm5
-; SSE2-NEXT: paddd %xmm1, %xmm5
-; SSE2-NEXT: paddd %xmm7, %xmm3
-; SSE2-NEXT: movdqa {{[-0-9]+}}(%r{{[sb]}}p), %xmm1 # 16-byte Reload
-; SSE2-NEXT: paddd %xmm8, %xmm1
+; SSE2-NEXT: paddd %xmm3, %xmm2
+; SSE2-NEXT: pxor %xmm5, %xmm5
+; SSE2-NEXT: paddd %xmm5, %xmm5
+; SSE2-NEXT: paddd %xmm3, %xmm0
; SSE2-NEXT: paddd %xmm3, %xmm1
+; SSE2-NEXT: paddd %xmm3, %xmm4
+; SSE2-NEXT: paddd %xmm5, %xmm4
; SSE2-NEXT: paddd %xmm5, %xmm1
-; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
-; SSE2-NEXT: paddd %xmm1, %xmm0
+; SSE2-NEXT: paddd %xmm4, %xmm1
+; SSE2-NEXT: paddd %xmm5, %xmm0
+; SSE2-NEXT: paddd %xmm2, %xmm5
+; SSE2-NEXT: paddd %xmm0, %xmm5
+; SSE2-NEXT: paddd %xmm1, %xmm5
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; SSE2-NEXT: paddd %xmm5, %xmm0
; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; SSE2-NEXT: paddd %xmm0, %xmm1
; SSE2-NEXT: movd %xmm1, %eax
@@ -417,40 +274,22 @@ define i32 @byte_sum_v64_i32() nounwind {
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpxor %xmm4, %xmm4, %xmm4
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
-; AVX2-NEXT: vpxor %xmm6, %xmm6, %xmm6
-; AVX2-NEXT: vpxor %xmm5, %xmm5, %xmm5
-; AVX2-NEXT: vpxor %xmm7, %xmm7, %xmm7
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB2_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm7, %ymm8, %ymm7
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm5, %ymm8, %ymm5
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm6, %ymm8, %ymm6
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm3, %ymm8, %ymm3
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm4, %ymm8, %ymm4
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm2, %ymm8, %ymm2
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm1, %ymm8, %ymm1
-; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm8 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
-; AVX2-NEXT: vpaddd %ymm0, %ymm8, %ymm0
+; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %ymm0, %ymm3
+; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
+; AVX2-NEXT: vpsadbw 1056(%rcx,%rax), %ymm0, %ymm3
+; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; AVX2-NEXT: addq $64, %rax
; AVX2-NEXT: jne .LBB2_1
; AVX2-NEXT: # %bb.2: # %middle.block
-; AVX2-NEXT: vpaddd %ymm6, %ymm0, %ymm0
-; AVX2-NEXT: vpaddd %ymm7, %ymm4, %ymm4
-; AVX2-NEXT: vpaddd %ymm4, %ymm0, %ymm0
-; AVX2-NEXT: vpaddd %ymm3, %ymm1, %ymm1
-; AVX2-NEXT: vpaddd %ymm5, %ymm2, %ymm2
-; AVX2-NEXT: vpaddd %ymm2, %ymm1, %ymm1
+; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm2
+; AVX2-NEXT: vpaddd %ymm0, %ymm0, %ymm3
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
+; AVX2-NEXT: vpaddd %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpaddd %ymm3, %ymm2, %ymm1
+; AVX2-NEXT: vpaddd %ymm1, %ymm0, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpaddd %xmm1, %xmm0, %xmm0
; AVX2-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
@@ -467,24 +306,16 @@ define i32 @byte_sum_v64_i32() nounwind {
; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
-; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX512BW-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB2_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm3, %zmm4, %zmm3
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm2, %zmm4, %zmm2
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm1, %zmm4, %zmm1
-; AVX512BW-NEXT: vpmovzxbd {{.*#+}} zmm4 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero,mem[8],zero,zero,zero,mem[9],zero,zero,zero,mem[10],zero,zero,zero,mem[11],zero,zero,zero,mem[12],zero,zero,zero,mem[13],zero,zero,zero,mem[14],zero,zero,zero,mem[15],zero,zero,zero
-; AVX512BW-NEXT: vpaddd %zmm0, %zmm4, %zmm0
+; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %zmm0, %zmm2
+; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1
; AVX512BW-NEXT: addq $64, %rax
; AVX512BW-NEXT: jne .LBB2_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
-; AVX512BW-NEXT: vpaddd %zmm2, %zmm1, %zmm1
-; AVX512BW-NEXT: vpaddd %zmm3, %zmm0, %zmm0
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm1
+; AVX512BW-NEXT: vpaddd %zmm0, %zmm0, %zmm0
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
; AVX512BW-NEXT: vpaddd %zmm1, %zmm0, %zmm0
@@ -522,63 +353,28 @@ middle.block:
define i64 @byte_sum_v16_i64() nounwind {
; SSE2-LABEL: byte_sum_v16_i64:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm4, %xmm4
+; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: pxor %xmm5, %xmm5
-; SSE2-NEXT: pxor %xmm7, %xmm7
-; SSE2-NEXT: pxor %xmm6, %xmm6
-; SSE2-NEXT: pxor %xmm8, %xmm8
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB3_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm10
-; SSE2-NEXT: movdqa %xmm10, %xmm9
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm9 = xmm9[8],xmm4[8],xmm9[9],xmm4[9],xmm9[10],xmm4[10],xmm9[11],xmm4[11],xmm9[12],xmm4[12],xmm9[13],xmm4[13],xmm9[14],xmm4[14],xmm9[15],xmm4[15]
-; SSE2-NEXT: movdqa %xmm9, %xmm11
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm11 = xmm11[4],xmm4[4],xmm11[5],xmm4[5],xmm11[6],xmm4[6],xmm11[7],xmm4[7]
-; SSE2-NEXT: movdqa %xmm11, %xmm12
-; SSE2-NEXT: punpckhdq {{.*#+}} xmm12 = xmm12[2],xmm4[2],xmm12[3],xmm4[3]
-; SSE2-NEXT: paddq %xmm12, %xmm8
-; SSE2-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1]
-; SSE2-NEXT: paddq %xmm11, %xmm6
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; SSE2-NEXT: movdqa %xmm9, %xmm11
-; SSE2-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm4[2],xmm11[3],xmm4[3]
-; SSE2-NEXT: paddq %xmm11, %xmm7
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm10 = xmm10[0],xmm4[0],xmm10[1],xmm4[1],xmm10[2],xmm4[2],xmm10[3],xmm4[3],xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; SSE2-NEXT: movdqa %xmm10, %xmm11
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1],xmm11[2],xmm4[2],xmm11[3],xmm4[3]
-; SSE2-NEXT: movdqa %xmm11, %xmm12
-; SSE2-NEXT: punpckldq {{.*#+}} xmm12 = xmm12[0],xmm4[0],xmm12[1],xmm4[1]
-; SSE2-NEXT: paddq %xmm12, %xmm1
-; SSE2-NEXT: punpckhdq {{.*#+}} xmm11 = xmm11[2],xmm4[2],xmm11[3],xmm4[3]
-; SSE2-NEXT: paddq %xmm11, %xmm3
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm10 = xmm10[4],xmm4[4],xmm10[5],xmm4[5],xmm10[6],xmm4[6],xmm10[7],xmm4[7]
-; SSE2-NEXT: movdqa %xmm10, %xmm11
-; SSE2-NEXT: punpckldq {{.*#+}} xmm11 = xmm11[0],xmm4[0],xmm11[1],xmm4[1]
-; SSE2-NEXT: paddq %xmm11, %xmm0
-; SSE2-NEXT: punpckhdq {{.*#+}} xmm10 = xmm10[2],xmm4[2],xmm10[3],xmm4[3]
-; SSE2-NEXT: paddq %xmm10, %xmm2
-; SSE2-NEXT: punpckldq {{.*#+}} xmm9 = xmm9[0],xmm4[0],xmm9[1],xmm4[1]
-; SSE2-NEXT: paddq %xmm9, %xmm5
+; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm2
+; SSE2-NEXT: psadbw %xmm0, %xmm2
+; SSE2-NEXT: paddq %xmm2, %xmm1
; SSE2-NEXT: addq $16, %rax
; SSE2-NEXT: jne .LBB3_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddq %xmm7, %xmm3
-; SSE2-NEXT: paddq %xmm8, %xmm2
-; SSE2-NEXT: paddq %xmm3, %xmm2
-; SSE2-NEXT: paddq %xmm5, %xmm1
-; SSE2-NEXT: paddq %xmm6, %xmm0
-; SSE2-NEXT: paddq %xmm1, %xmm0
-; SSE2-NEXT: paddq %xmm2, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
; SSE2-NEXT: paddq %xmm0, %xmm1
-; SSE2-NEXT: movq %xmm1, %rax
+; SSE2-NEXT: paddq %xmm0, %xmm0
+; SSE2-NEXT: movdqa %xmm0, %xmm2
+; SSE2-NEXT: paddq %xmm0, %xmm2
+; SSE2-NEXT: paddq %xmm0, %xmm2
+; SSE2-NEXT: paddq %xmm1, %xmm2
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; SSE2-NEXT: paddq %xmm2, %xmm0
+; SSE2-NEXT: movq %xmm0, %rax
; SSE2-NEXT: retq
;
; AVX2-LABEL: byte_sum_v16_i64:
@@ -588,23 +384,16 @@ define i64 @byte_sum_v16_i64() nounwind {
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
-; AVX2-NEXT: vpxor %xmm3, %xmm3, %xmm3
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB3_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpaddq %ymm3, %ymm4, %ymm3
-; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpaddq %ymm2, %ymm4, %ymm2
-; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpaddq %ymm1, %ymm4, %ymm1
-; AVX2-NEXT: vpmovzxbq {{.*#+}} ymm4 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero
-; AVX2-NEXT: vpaddq %ymm0, %ymm4, %ymm0
+; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm3
+; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm2
; AVX2-NEXT: addq $16, %rax
; AVX2-NEXT: jne .LBB3_1
; AVX2-NEXT: # %bb.2: # %middle.block
-; AVX2-NEXT: vpaddq %ymm2, %ymm1, %ymm1
-; AVX2-NEXT: vpaddq %ymm3, %ymm0, %ymm0
+; AVX2-NEXT: vpaddq %ymm0, %ymm2, %ymm1
+; AVX2-NEXT: vpaddq %ymm0, %ymm0, %ymm0
; AVX2-NEXT: vpaddq %ymm0, %ymm1, %ymm0
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
; AVX2-NEXT: vpaddq %xmm1, %xmm0, %xmm0
@@ -620,17 +409,16 @@ define i64 @byte_sum_v16_i64() nounwind {
; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
+; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB3_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpmovzxbq {{.*#+}} zmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT: vpaddq %zmm1, %zmm2, %zmm1
-; AVX512BW-NEXT: vpmovzxbq {{.*#+}} zmm2 = mem[0],zero,zero,zero,zero,zero,zero,zero,mem[1],zero,zero,zero,zero,zero,zero,zero,mem[2],zero,zero,zero,zero,zero,zero,zero,mem[3],zero,zero,zero,zero,zero,zero,zero,mem[4],zero,zero,zero,zero,zero,zero,zero,mem[5],zero,zero,zero,zero,zero,zero,zero,mem[6],zero,zero,zero,zero,zero,zero,zero,mem[7],zero,zero,zero,zero,zero,zero,zero
-; AVX512BW-NEXT: vpaddq %zmm0, %zmm2, %zmm0
+; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm3
+; AVX512BW-NEXT: vpaddq %zmm2, %zmm3, %zmm2
; AVX512BW-NEXT: addq $16, %rax
; AVX512BW-NEXT: jne .LBB3_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
-; AVX512BW-NEXT: vpaddq %zmm0, %zmm1, %zmm0
+; AVX512BW-NEXT: vpaddq %zmm0, %zmm2, %zmm0
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
; AVX512BW-NEXT: vpaddq %zmm1, %zmm0, %zmm0
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll
new file mode 100644
index 0000000000000..fcd757ba163ad
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll
@@ -0,0 +1,43 @@
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+sse2 -S | FileCheck %s
+
+; Verify that X86PartialReduction::tryByteSumReplacement carries the
+; original add instruction's !dbg location onto the freshly emitted
+; psadbw / shuffle sequence so source locations survive into the
+; optimized IR under -g.
+
+ at a = global [1024 x i8] zeroinitializer, align 16
+
+; CHECK-LABEL: @byte_sum_v16_i32
+; CHECK: call <2 x i64> @llvm.x86.sse2.psad.bw({{.*}}), !dbg ![[#LOC:]]
+; CHECK: ![[#LOC]] = !DILocation(line: 42,
+define i32 @byte_sum_v16_i32() nounwind !dbg !6 {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <16 x i8>, ptr %p, align 16
+ %z = zext <16 x i8> %wide.load to <16 x i32>, !dbg !8
+ %add = add nsw <16 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
+ ret i32 %ext
+}
+
+!llvm.dbg.cu = !{!0}
+!llvm.module.flags = !{!3, !4, !5}
+
+!0 = distinct !DICompileUnit(language: DW_LANG_C99, file: !1, producer: "clang", isOptimized: true, runtimeVersion: 0, emissionKind: FullDebug)
+!1 = !DIFile(filename: "byte-sum-debugloc.c", directory: "/tmp")
+!3 = !{i32 7, !"Dwarf Version", i32 4}
+!4 = !{i32 2, !"Debug Info Version", i32 3}
+!5 = !{i32 1, !"wchar_size", i32 4}
+!6 = distinct !DISubprogram(name: "byte_sum_v16_i32", scope: !1, file: !1, line: 1, type: !7, scopeLine: 1, flags: DIFlagPrototyped, spFlags: DISPFlagDefinition, unit: !0)
+!7 = !DISubroutineType(types: !{})
+!8 = !DILocation(line: 42, column: 1, scope: !6)
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll
new file mode 100644
index 0000000000000..d46edfd754514
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll
@@ -0,0 +1,49 @@
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+avx2 -S | FileCheck %s
+
+; Shapes that tryByteSumReplacement must not rewrite.
+
+ at a = global [1024 x i8] zeroinitializer, align 16
+
+; CHECK-LABEL: @byte_sum_v8_i32
+; CHECK-NOT: psad.bw
+define i32 @byte_sum_v8_i32() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <8 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <8 x i8>, ptr %p, align 8
+ %z = zext <8 x i8> %wide.load to <8 x i32>
+ %add = add nsw <8 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 8
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %add)
+ ret i32 %ext
+}
+
+; CHECK-LABEL: @byte_sum_v24_i32
+; CHECK-NOT: psad.bw
+define i32 @byte_sum_v24_i32() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <24 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <24 x i8>, ptr %p, align 8
+ %z = zext <24 x i8> %wide.load to <24 x i32>
+ %add = add nsw <24 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 24
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %add)
+ ret i32 %ext
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll
new file mode 100644
index 0000000000000..23be1178bf767
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll
@@ -0,0 +1,59 @@
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+avx2 -S | FileCheck %s --check-prefix=AVX2
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+avx512bw -S | FileCheck %s --check-prefix=AVX512
+
+; Wider-VF i64-accumulator positive shapes for tryByteSumReplacement.
+; These exercise the PerSplitTy = <IntrinsicNumElts/8 x i64> branch where the
+; matcher must NOT bitcast the psadbw result back to i32.
+;
+; VF=32 i64 uses the AVX2 256-bit lane (one avx2.psad.bw call). On
+; +avx512bw the dispatch falls into the same AVX2 path because NumElts<64.
+;
+; VF=64 i64 uses the AVX-512BW 512-bit lane (one avx512.psad.bw.512 call).
+
+ at a = global [1024 x i8] zeroinitializer, align 16
+
+; AVX2-LABEL: @byte_sum_v32_i64(
+; AVX2: call <4 x i64> @llvm.x86.avx2.psad.bw(
+; AVX512-LABEL: @byte_sum_v32_i64(
+; AVX512: call <4 x i64> @llvm.x86.avx2.psad.bw(
+define i64 @byte_sum_v32_i64() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <32 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <32 x i8>, ptr %p, align 16
+ %z = zext <32 x i8> %wide.load to <32 x i64>
+ %add = add nsw <32 x i64> %z, %vec.phi
+ %index.next = add i64 %index, 32
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i64 @llvm.vector.reduce.add.v32i64(<32 x i64> %add)
+ ret i64 %ext
+}
+
+; AVX512-LABEL: @byte_sum_v64_i64(
+; AVX512: call <8 x i64> @llvm.x86.avx512.psad.bw.512(
+define i64 @byte_sum_v64_i64() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <64 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <64 x i8>, ptr %p, align 16
+ %z = zext <64 x i8> %wide.load to <64 x i64>
+ %add = add nsw <64 x i64> %z, %vec.phi
+ %index.next = add i64 %index, 64
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i64 @llvm.vector.reduce.add.v64i64(<64 x i64> %add)
+ ret i64 %ext
+}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
new file mode 100644
index 0000000000000..6acd8de044aa2
--- /dev/null
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
@@ -0,0 +1,26 @@
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+sse2 -S | FileCheck %s
+
+; Isolate X86PartialReduction::tryByteSumReplacement on a positive shape.
+
+ at a = global [1024 x i8] zeroinitializer, align 16
+
+; CHECK: call <2 x i64> @llvm.x86.sse2.psad.bw(
+define i32 @byte_sum_v16_i32() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <16 x i8>, ptr %p, align 16
+ %z = zext <16 x i8> %wide.load to <16 x i32>
+ %add = add nsw <16 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 16
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
+ ret i32 %ext
+}
>From 4f9d3918aab73b1f66b64a3bffbe2a775d46cf24 Mon Sep 17 00:00:00 2001
From: mbhade <mbhade at amd.com>
Date: Thu, 6 Aug 2026 17:47:29 +0530
Subject: [PATCH 3/3] Merge opt tests and add x86-64-v2/v3/v4 coverage
---
llvm/test/CodeGen/X86/byte-sum.ll | 501 ++++++++++++++----
...x86-partial-reduction-byte-sum-debugloc.ll | 2 +-
...x86-partial-reduction-byte-sum-negative.ll | 49 --
...x86-partial-reduction-byte-sum-wide-i64.ll | 59 ---
.../X86/x86-partial-reduction-byte-sum.ll | 110 +++-
5 files changed, 516 insertions(+), 205 deletions(-)
delete mode 100644 llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll
delete mode 100644 llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll
diff --git a/llvm/test/CodeGen/X86/byte-sum.ll b/llvm/test/CodeGen/X86/byte-sum.ll
index 0a22fe31b46f7..85988357d55ac 100644
--- a/llvm/test/CodeGen/X86/byte-sum.ll
+++ b/llvm/test/CodeGen/X86/byte-sum.ll
@@ -1,7 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+sse2 | FileCheck %s --check-prefixes=SSE2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2
-; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx512bw | FileCheck %s --check-prefixes=AVX512BW
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64 | FileCheck %s --check-prefixes=SSE2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 | FileCheck %s --check-prefixes=SSE42
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 | FileCheck %s --check-prefixes=AVX2
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 | FileCheck %s --check-prefixes=AVX512BW
; CodeGen tests for X86PartialReduction::tryByteSumReplacement.
; Positive cases should lower to vpsadbw; negative cases should not.
@@ -15,16 +16,17 @@ define i32 @byte_sum_v16_i32() nounwind {
; SSE2-LABEL: byte_sum_v16_i32:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB0_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm2
+; SSE2-NEXT: movdqa (%rcx,%rax), %xmm2
; SSE2-NEXT: psadbw %xmm0, %xmm2
; SSE2-NEXT: paddd %xmm2, %xmm1
; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB0_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm0, %xmm1
@@ -37,19 +39,46 @@ define i32 @byte_sum_v16_i32() nounwind {
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v16_i32:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB0_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: movdqa (%rcx,%rax), %xmm2
+; SSE42-NEXT: psadbw %xmm0, %xmm2
+; SSE42-NEXT: paddd %xmm2, %xmm1
+; SSE42-NEXT: addq $16, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB0_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: paddd %xmm0, %xmm0
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: movd %xmm0, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v16_i32:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB0_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm3
+; AVX2-NEXT: vpsadbw (%rcx,%rax), %xmm1, %xmm3
; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB0_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
@@ -66,15 +95,16 @@ define i32 @byte_sum_v16_i32() nounwind {
; AVX512BW-LABEL: byte_sum_v16_i32:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB0_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm2
+; AVX512BW-NEXT: vpsadbw (%rcx,%rax), %xmm1, %xmm2
; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB0_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
@@ -114,20 +144,21 @@ define i32 @byte_sum_v32_i32() nounwind {
; SSE2-LABEL: byte_sum_v32_i32:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB1_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm3
+; SSE2-NEXT: movdqa (%rcx,%rax), %xmm3
; SSE2-NEXT: psadbw %xmm0, %xmm3
; SSE2-NEXT: paddd %xmm3, %xmm1
-; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm3
+; SSE2-NEXT: movdqa 16(%rcx,%rax), %xmm3
; SSE2-NEXT: psadbw %xmm0, %xmm3
; SSE2-NEXT: paddd %xmm3, %xmm2
; SSE2-NEXT: addq $32, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB1_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm0, %xmm2
@@ -143,18 +174,52 @@ define i32 @byte_sum_v32_i32() nounwind {
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v32_i32:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB1_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: movdqa (%rcx,%rax), %xmm3
+; SSE42-NEXT: psadbw %xmm0, %xmm3
+; SSE42-NEXT: paddd %xmm3, %xmm1
+; SSE42-NEXT: movdqa 16(%rcx,%rax), %xmm3
+; SSE42-NEXT: psadbw %xmm0, %xmm3
+; SSE42-NEXT: paddd %xmm3, %xmm2
+; SSE42-NEXT: addq $32, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB1_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm0, %xmm2
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: paddd %xmm0, %xmm0
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: paddd %xmm2, %xmm0
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: movd %xmm0, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v32_i32:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB1_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %ymm0, %ymm2
+; AVX2-NEXT: vpsadbw (%rcx,%rax), %ymm0, %ymm2
; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
; AVX2-NEXT: addq $32, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB1_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm1
@@ -173,16 +238,17 @@ define i32 @byte_sum_v32_i32() nounwind {
; AVX512BW-LABEL: byte_sum_v32_i32:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB1_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %ymm1, %ymm3
+; AVX512BW-NEXT: vpsadbw (%rcx,%rax), %ymm1, %ymm3
; AVX512BW-NEXT: vpaddd %zmm2, %zmm3, %zmm2
; AVX512BW-NEXT: addq $32, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB1_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vpaddd %zmm0, %zmm2, %zmm0
@@ -223,7 +289,7 @@ define i32 @byte_sum_v64_i32() nounwind {
; SSE2-LABEL: byte_sum_v64_i32:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm3, %xmm3
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pxor %xmm4, %xmm4
@@ -232,19 +298,20 @@ define i32 @byte_sum_v64_i32() nounwind {
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB2_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm5
+; SSE2-NEXT: movdqa (%rcx,%rax), %xmm5
; SSE2-NEXT: psadbw %xmm3, %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm0
-; SSE2-NEXT: movdqa 1040(%rcx,%rax), %xmm5
+; SSE2-NEXT: movdqa 16(%rcx,%rax), %xmm5
; SSE2-NEXT: psadbw %xmm3, %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm4
-; SSE2-NEXT: movdqa 1056(%rcx,%rax), %xmm5
+; SSE2-NEXT: movdqa 32(%rcx,%rax), %xmm5
; SSE2-NEXT: psadbw %xmm3, %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm2
-; SSE2-NEXT: movdqa 1072(%rcx,%rax), %xmm5
+; SSE2-NEXT: movdqa 48(%rcx,%rax), %xmm5
; SSE2-NEXT: psadbw %xmm3, %xmm5
; SSE2-NEXT: paddd %xmm5, %xmm1
; SSE2-NEXT: addq $64, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB2_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm3, %xmm2
@@ -267,21 +334,70 @@ define i32 @byte_sum_v64_i32() nounwind {
; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v64_i32:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: pxor %xmm4, %xmm4
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB2_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: movdqa (%rcx,%rax), %xmm5
+; SSE42-NEXT: psadbw %xmm3, %xmm5
+; SSE42-NEXT: paddd %xmm5, %xmm0
+; SSE42-NEXT: movdqa 16(%rcx,%rax), %xmm5
+; SSE42-NEXT: psadbw %xmm3, %xmm5
+; SSE42-NEXT: paddd %xmm5, %xmm4
+; SSE42-NEXT: movdqa 32(%rcx,%rax), %xmm5
+; SSE42-NEXT: psadbw %xmm3, %xmm5
+; SSE42-NEXT: paddd %xmm5, %xmm2
+; SSE42-NEXT: movdqa 48(%rcx,%rax), %xmm5
+; SSE42-NEXT: psadbw %xmm3, %xmm5
+; SSE42-NEXT: paddd %xmm5, %xmm1
+; SSE42-NEXT: addq $64, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB2_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm3, %xmm2
+; SSE42-NEXT: pxor %xmm5, %xmm5
+; SSE42-NEXT: paddd %xmm5, %xmm5
+; SSE42-NEXT: paddd %xmm3, %xmm0
+; SSE42-NEXT: paddd %xmm3, %xmm1
+; SSE42-NEXT: paddd %xmm3, %xmm4
+; SSE42-NEXT: paddd %xmm5, %xmm4
+; SSE42-NEXT: paddd %xmm5, %xmm1
+; SSE42-NEXT: paddd %xmm4, %xmm1
+; SSE42-NEXT: paddd %xmm5, %xmm0
+; SSE42-NEXT: paddd %xmm2, %xmm5
+; SSE42-NEXT: paddd %xmm0, %xmm5
+; SSE42-NEXT: paddd %xmm1, %xmm5
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm5[2,3,2,3]
+; SSE42-NEXT: paddd %xmm5, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v64_i32:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB2_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %ymm0, %ymm3
+; AVX2-NEXT: vpsadbw (%rcx,%rax), %ymm0, %ymm3
+; AVX2-NEXT: vpsadbw 32(%rcx,%rax), %ymm0, %ymm4
; AVX2-NEXT: vpaddd %ymm1, %ymm3, %ymm1
-; AVX2-NEXT: vpsadbw 1056(%rcx,%rax), %ymm0, %ymm3
-; AVX2-NEXT: vpaddd %ymm2, %ymm3, %ymm2
+; AVX2-NEXT: vpaddd %ymm2, %ymm4, %ymm2
; AVX2-NEXT: addq $64, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB2_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm2
@@ -303,15 +419,16 @@ define i32 @byte_sum_v64_i32() nounwind {
; AVX512BW-LABEL: byte_sum_v64_i32:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB2_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %zmm0, %zmm2
+; AVX512BW-NEXT: vpsadbw (%rcx,%rax), %zmm0, %zmm2
; AVX512BW-NEXT: vpaddd %zmm1, %zmm2, %zmm1
; AVX512BW-NEXT: addq $64, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB2_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm1
@@ -354,16 +471,17 @@ define i64 @byte_sum_v16_i64() nounwind {
; SSE2-LABEL: byte_sum_v16_i64:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB3_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm2
+; SSE2-NEXT: movdqa (%rcx,%rax), %xmm2
; SSE2-NEXT: psadbw %xmm0, %xmm2
; SSE2-NEXT: paddq %xmm2, %xmm1
; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB3_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddq %xmm0, %xmm1
@@ -377,19 +495,47 @@ define i64 @byte_sum_v16_i64() nounwind {
; SSE2-NEXT: movq %xmm0, %rax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v16_i64:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB3_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: movdqa (%rcx,%rax), %xmm2
+; SSE42-NEXT: psadbw %xmm0, %xmm2
+; SSE42-NEXT: paddq %xmm2, %xmm1
+; SSE42-NEXT: addq $16, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB3_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddq %xmm0, %xmm1
+; SSE42-NEXT: paddq %xmm0, %xmm0
+; SSE42-NEXT: movdqa %xmm0, %xmm2
+; SSE42-NEXT: paddq %xmm0, %xmm2
+; SSE42-NEXT: paddq %xmm0, %xmm2
+; SSE42-NEXT: paddq %xmm1, %xmm2
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm2[2,3,2,3]
+; SSE42-NEXT: paddq %xmm2, %xmm0
+; SSE42-NEXT: movq %xmm0, %rax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v16_i64:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB3_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm3
+; AVX2-NEXT: vpsadbw (%rcx,%rax), %xmm1, %xmm3
; AVX2-NEXT: vpaddq %ymm2, %ymm3, %ymm2
; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB3_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddq %ymm0, %ymm2, %ymm1
@@ -406,16 +552,17 @@ define i64 @byte_sum_v16_i64() nounwind {
; AVX512BW-LABEL: byte_sum_v16_i64:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX512BW-NEXT: vpxor %xmm2, %xmm2, %xmm2
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB3_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpsadbw 1024(%rcx,%rax), %xmm1, %xmm3
+; AVX512BW-NEXT: vpsadbw (%rcx,%rax), %xmm1, %xmm3
; AVX512BW-NEXT: vpaddq %zmm2, %zmm3, %zmm2
; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB3_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vpaddq %zmm0, %zmm2, %zmm0
@@ -454,7 +601,7 @@ define i32 @byte_sum_v16_i32_sext() nounwind {
; SSE2-LABEL: byte_sum_v16_i32_sext:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm3, %xmm3
@@ -462,7 +609,7 @@ define i32 @byte_sum_v16_i32_sext() nounwind {
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB4_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm4
+; SSE2-NEXT: movdqa (%rcx,%rax), %xmm4
; SSE2-NEXT: punpcklbw {{.*#+}} xmm5 = xmm5[0],xmm4[0],xmm5[1],xmm4[1],xmm5[2],xmm4[2],xmm5[3],xmm4[3],xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm5[0],xmm6[1],xmm5[1],xmm6[2],xmm5[2],xmm6[3],xmm5[3]
; SSE2-NEXT: psrad $24, %xmm6
@@ -478,6 +625,7 @@ define i32 @byte_sum_v16_i32_sext() nounwind {
; SSE2-NEXT: psrad $24, %xmm4
; SSE2-NEXT: paddd %xmm4, %xmm2
; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB4_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm3, %xmm0
@@ -490,20 +638,54 @@ define i32 @byte_sum_v16_i32_sext() nounwind {
; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v16_i32_sext:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB4_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: pmovsxbd 12(%rcx,%rax), %xmm4
+; SSE42-NEXT: paddd %xmm4, %xmm2
+; SSE42-NEXT: pmovsxbd 8(%rcx,%rax), %xmm4
+; SSE42-NEXT: paddd %xmm4, %xmm3
+; SSE42-NEXT: pmovsxbd (%rcx,%rax), %xmm4
+; SSE42-NEXT: pmovsxbd 4(%rcx,%rax), %xmm5
+; SSE42-NEXT: paddd %xmm4, %xmm0
+; SSE42-NEXT: paddd %xmm5, %xmm1
+; SSE42-NEXT: addq $16, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB4_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm3, %xmm0
+; SSE42-NEXT: paddd %xmm2, %xmm1
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v16_i32_sext:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB4_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpmovsxbd 1024(%rcx,%rax), %ymm2
+; AVX2-NEXT: vpmovsxbd (%rcx,%rax), %ymm2
+; AVX2-NEXT: vpmovsxbd 8(%rcx,%rax), %ymm3
; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpmovsxbd 1032(%rcx,%rax), %ymm2
-; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
+; AVX2-NEXT: vpaddd %ymm0, %ymm3, %ymm0
; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB4_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
@@ -520,14 +702,15 @@ define i32 @byte_sum_v16_i32_sext() nounwind {
; AVX512BW-LABEL: byte_sum_v16_i32_sext:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB4_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpmovsxbd 1024(%rcx,%rax), %zmm1
+; AVX512BW-NEXT: vpmovsxbd (%rcx,%rax), %zmm1
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB4_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
@@ -567,7 +750,7 @@ define i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; SSE2-LABEL: word_sum_v16_i32:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm2, %xmm2
-; SSE2-NEXT: movq $-2048, %rax # imm = 0xF800
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm0, %xmm0
; SSE2-NEXT: pxor %xmm4, %xmm4
@@ -575,8 +758,8 @@ define i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB5_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqu 2048(%rdi,%rax), %xmm5
-; SSE2-NEXT: movdqu 2064(%rdi,%rax), %xmm6
+; SSE2-NEXT: movdqu (%rdi,%rax,2), %xmm5
+; SSE2-NEXT: movdqu 16(%rdi,%rax,2), %xmm6
; SSE2-NEXT: movdqa %xmm6, %xmm7
; SSE2-NEXT: punpckhwd {{.*#+}} xmm7 = xmm7[4],xmm2[4],xmm7[5],xmm2[5],xmm7[6],xmm2[6],xmm7[7],xmm2[7]
; SSE2-NEXT: paddd %xmm7, %xmm3
@@ -587,7 +770,8 @@ define i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; SSE2-NEXT: paddd %xmm5, %xmm0
; SSE2-NEXT: punpcklwd {{.*#+}} xmm6 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
; SSE2-NEXT: paddd %xmm6, %xmm4
-; SSE2-NEXT: addq $32, %rax
+; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB5_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm4, %xmm1
@@ -600,19 +784,52 @@ define i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: word_sum_v16_i32:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB5_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; SSE42-NEXT: paddd %xmm4, %xmm2
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; SSE42-NEXT: paddd %xmm4, %xmm3
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm4 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; SSE42-NEXT: pmovzxwd {{.*#+}} xmm5 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero
+; SSE42-NEXT: paddd %xmm4, %xmm0
+; SSE42-NEXT: paddd %xmm5, %xmm1
+; SSE42-NEXT: addq $16, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB5_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm3, %xmm0
+; SSE42-NEXT: paddd %xmm2, %xmm1
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: word_sum_v16_i32:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-2048, %rax # imm = 0xF800
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB5_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
; AVX2-NEXT: vpaddd %ymm1, %ymm2, %ymm1
-; AVX2-NEXT: vpmovzxwd {{.*#+}} ymm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
-; AVX2-NEXT: vpaddd %ymm0, %ymm2, %ymm0
-; AVX2-NEXT: addq $32, %rax
+; AVX2-NEXT: vpaddd %ymm0, %ymm3, %ymm0
+; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB5_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
@@ -629,13 +846,14 @@ define i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; AVX512BW-LABEL: word_sum_v16_i32:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-2048, %rax # imm = 0xF800
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB5_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
; AVX512BW-NEXT: vpmovzxwd {{.*#+}} zmm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
; AVX512BW-NEXT: vpaddd %zmm0, %zmm1, %zmm0
-; AVX512BW-NEXT: addq $32, %rax
+; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB5_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
@@ -674,39 +892,69 @@ middle.block:
define i16 @byte_sum_v16_i16() nounwind {
; SSE2-LABEL: byte_sum_v16_i16:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
-; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: xorl %eax, %eax
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB6_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqa 1024(%rcx,%rax), %xmm3
+; SSE2-NEXT: movdqa (%rcx,%rax), %xmm3
; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3],xmm4[4],xmm1[4],xmm4[5],xmm1[5],xmm4[6],xmm1[6],xmm4[7],xmm1[7]
-; SSE2-NEXT: paddw %xmm4, %xmm0
-; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm1[8],xmm3[9],xmm1[9],xmm3[10],xmm1[10],xmm3[11],xmm1[11],xmm3[12],xmm1[12],xmm3[13],xmm1[13],xmm3[14],xmm1[14],xmm3[15],xmm1[15]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3],xmm4[4],xmm0[4],xmm4[5],xmm0[5],xmm4[6],xmm0[6],xmm4[7],xmm0[7]
+; SSE2-NEXT: paddw %xmm4, %xmm1
+; SSE2-NEXT: punpckhbw {{.*#+}} xmm3 = xmm3[8],xmm0[8],xmm3[9],xmm0[9],xmm3[10],xmm0[10],xmm3[11],xmm0[11],xmm3[12],xmm0[12],xmm3[13],xmm0[13],xmm3[14],xmm0[14],xmm3[15],xmm0[15]
; SSE2-NEXT: paddw %xmm3, %xmm2
; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB6_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddw %xmm2, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE2-NEXT: paddw %xmm0, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddw %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; SSE2-NEXT: paddw %xmm1, %xmm0
-; SSE2-NEXT: movdqa %xmm0, %xmm1
-; SSE2-NEXT: psrld $16, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
; SSE2-NEXT: paddw %xmm0, %xmm1
-; SSE2-NEXT: movd %xmm1, %eax
+; SSE2-NEXT: movdqa %xmm1, %xmm0
+; SSE2-NEXT: psrld $16, %xmm0
+; SSE2-NEXT: paddw %xmm1, %xmm0
+; SSE2-NEXT: movd %xmm0, %eax
; SSE2-NEXT: # kill: def $ax killed $ax killed $eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v16_i16:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB6_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: pmovzxbw {{.*#+}} xmm2 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; SSE42-NEXT: pmovzxbw {{.*#+}} xmm3 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero
+; SSE42-NEXT: paddw %xmm2, %xmm0
+; SSE42-NEXT: paddw %xmm3, %xmm1
+; SSE42-NEXT: addq $16, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB6_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddw %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE42-NEXT: paddw %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: paddw %xmm1, %xmm0
+; SSE42-NEXT: movdqa %xmm0, %xmm1
+; SSE42-NEXT: psrld $16, %xmm1
+; SSE42-NEXT: paddw %xmm0, %xmm1
+; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: # kill: def $ax killed $ax killed $eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v16_i16:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB6_1: # %vector.body
@@ -714,6 +962,7 @@ define i16 @byte_sum_v16_i16() nounwind {
; AVX2-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
; AVX2-NEXT: vpaddw %ymm0, %ymm1, %ymm0
; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB6_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
@@ -732,7 +981,7 @@ define i16 @byte_sum_v16_i16() nounwind {
; AVX512BW-LABEL: byte_sum_v16_i16:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB6_1: # %vector.body
@@ -740,6 +989,7 @@ define i16 @byte_sum_v16_i16() nounwind {
; AVX512BW-NEXT: vpmovzxbw {{.*#+}} ymm1 = mem[0],zero,mem[1],zero,mem[2],zero,mem[3],zero,mem[4],zero,mem[5],zero,mem[6],zero,mem[7],zero,mem[8],zero,mem[9],zero,mem[10],zero,mem[11],zero,mem[12],zero,mem[13],zero,mem[14],zero,mem[15],zero
; AVX512BW-NEXT: vpaddw %ymm0, %ymm1, %ymm0
; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB6_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
@@ -780,22 +1030,23 @@ define i32 @dword_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; SSE2-LABEL: dword_sum_v16_i32:
; SSE2: # %bb.0: # %entry
; SSE2-NEXT: pxor %xmm0, %xmm0
-; SSE2-NEXT: movq $-4096, %rax # imm = 0xF000
+; SSE2-NEXT: xorl %eax, %eax
; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm3, %xmm3
; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB7_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
-; SSE2-NEXT: movdqu 4096(%rdi,%rax), %xmm4
+; SSE2-NEXT: movdqu (%rdi,%rax,4), %xmm4
; SSE2-NEXT: paddd %xmm4, %xmm0
-; SSE2-NEXT: movdqu 4112(%rdi,%rax), %xmm4
+; SSE2-NEXT: movdqu 16(%rdi,%rax,4), %xmm4
; SSE2-NEXT: paddd %xmm4, %xmm1
-; SSE2-NEXT: movdqu 4128(%rdi,%rax), %xmm4
+; SSE2-NEXT: movdqu 32(%rdi,%rax,4), %xmm4
; SSE2-NEXT: paddd %xmm4, %xmm3
-; SSE2-NEXT: movdqu 4144(%rdi,%rax), %xmm4
+; SSE2-NEXT: movdqu 48(%rdi,%rax,4), %xmm4
; SSE2-NEXT: paddd %xmm4, %xmm2
-; SSE2-NEXT: addq $64, %rax
+; SSE2-NEXT: addq $16, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB7_1
; SSE2-NEXT: # %bb.2: # %middle.block
; SSE2-NEXT: paddd %xmm3, %xmm0
@@ -808,17 +1059,50 @@ define i32 @dword_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: dword_sum_v16_i32:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: pxor %xmm3, %xmm3
+; SSE42-NEXT: pxor %xmm2, %xmm2
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB7_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: movdqu (%rdi,%rax,4), %xmm4
+; SSE42-NEXT: paddd %xmm4, %xmm0
+; SSE42-NEXT: movdqu 16(%rdi,%rax,4), %xmm4
+; SSE42-NEXT: paddd %xmm4, %xmm1
+; SSE42-NEXT: movdqu 32(%rdi,%rax,4), %xmm4
+; SSE42-NEXT: paddd %xmm4, %xmm3
+; SSE42-NEXT: movdqu 48(%rdi,%rax,4), %xmm4
+; SSE42-NEXT: paddd %xmm4, %xmm2
+; SSE42-NEXT: addq $16, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB7_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm3, %xmm0
+; SSE42-NEXT: paddd %xmm2, %xmm1
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: movd %xmm1, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: dword_sum_v16_i32:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-4096, %rax # imm = 0xF000
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: vpxor %xmm1, %xmm1, %xmm1
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB7_1: # %vector.body
; AVX2-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX2-NEXT: vpaddd 4096(%rdi,%rax), %ymm1, %ymm1
-; AVX2-NEXT: vpaddd 4128(%rdi,%rax), %ymm0, %ymm0
-; AVX2-NEXT: addq $64, %rax
+; AVX2-NEXT: vpaddd (%rdi,%rax,4), %ymm1, %ymm1
+; AVX2-NEXT: vpaddd 32(%rdi,%rax,4), %ymm0, %ymm0
+; AVX2-NEXT: addq $16, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB7_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
@@ -835,12 +1119,13 @@ define i32 @dword_sum_v16_i32(ptr nocapture readonly %src) nounwind {
; AVX512BW-LABEL: dword_sum_v16_i32:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-4096, %rax # imm = 0xF000
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB7_1: # %vector.body
; AVX512BW-NEXT: # =>This Inner Loop Header: Depth=1
-; AVX512BW-NEXT: vpaddd 4096(%rdi,%rax), %zmm0, %zmm0
-; AVX512BW-NEXT: addq $64, %rax
+; AVX512BW-NEXT: vpaddd (%rdi,%rax,4), %zmm0, %zmm0
+; AVX512BW-NEXT: addq $16, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB7_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vextracti64x4 $1, %zmm0, %ymm1
@@ -878,36 +1163,62 @@ middle.block:
define i32 @byte_sum_v8_i32() nounwind {
; SSE2-LABEL: byte_sum_v8_i32:
; SSE2: # %bb.0: # %entry
-; SSE2-NEXT: pxor %xmm1, %xmm1
-; SSE2-NEXT: movq $-1024, %rax # imm = 0xFC00
-; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
; SSE2-NEXT: pxor %xmm0, %xmm0
+; SSE2-NEXT: xorl %eax, %eax
+; SSE2-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE2-NEXT: pxor %xmm1, %xmm1
; SSE2-NEXT: pxor %xmm2, %xmm2
; SSE2-NEXT: .p2align 4
; SSE2-NEXT: .LBB8_1: # %vector.body
; SSE2-NEXT: # =>This Inner Loop Header: Depth=1
; SSE2-NEXT: movq {{.*#+}} xmm3 = mem[0],zero
-; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm1[0],xmm3[1],xmm1[1],xmm3[2],xmm1[2],xmm3[3],xmm1[3],xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT: punpcklbw {{.*#+}} xmm3 = xmm3[0],xmm0[0],xmm3[1],xmm0[1],xmm3[2],xmm0[2],xmm3[3],xmm0[3],xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
; SSE2-NEXT: movdqa %xmm3, %xmm4
-; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm1[0],xmm4[1],xmm1[1],xmm4[2],xmm1[2],xmm4[3],xmm1[3]
-; SSE2-NEXT: paddd %xmm4, %xmm0
-; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm1[4],xmm3[5],xmm1[5],xmm3[6],xmm1[6],xmm3[7],xmm1[7]
+; SSE2-NEXT: punpcklwd {{.*#+}} xmm4 = xmm4[0],xmm0[0],xmm4[1],xmm0[1],xmm4[2],xmm0[2],xmm4[3],xmm0[3]
+; SSE2-NEXT: paddd %xmm4, %xmm1
+; SSE2-NEXT: punpckhwd {{.*#+}} xmm3 = xmm3[4],xmm0[4],xmm3[5],xmm0[5],xmm3[6],xmm0[6],xmm3[7],xmm0[7]
; SSE2-NEXT: paddd %xmm3, %xmm2
; SSE2-NEXT: addq $8, %rax
+; SSE2-NEXT: cmpq $1024, %rax # imm = 0x400
; SSE2-NEXT: jne .LBB8_1
; SSE2-NEXT: # %bb.2: # %middle.block
-; SSE2-NEXT: paddd %xmm2, %xmm0
-; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
-; SSE2-NEXT: paddd %xmm0, %xmm1
-; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE2-NEXT: paddd %xmm2, %xmm1
+; SSE2-NEXT: pshufd {{.*#+}} xmm0 = xmm1[2,3,2,3]
; SSE2-NEXT: paddd %xmm1, %xmm0
-; SSE2-NEXT: movd %xmm0, %eax
+; SSE2-NEXT: pshufd {{.*#+}} xmm1 = xmm0[1,1,1,1]
+; SSE2-NEXT: paddd %xmm0, %xmm1
+; SSE2-NEXT: movd %xmm1, %eax
; SSE2-NEXT: retq
;
+; SSE42-LABEL: byte_sum_v8_i32:
+; SSE42: # %bb.0: # %entry
+; SSE42-NEXT: pxor %xmm0, %xmm0
+; SSE42-NEXT: xorl %eax, %eax
+; SSE42-NEXT: movq a at GOTPCREL(%rip), %rcx
+; SSE42-NEXT: pxor %xmm1, %xmm1
+; SSE42-NEXT: .p2align 4
+; SSE42-NEXT: .LBB8_1: # %vector.body
+; SSE42-NEXT: # =>This Inner Loop Header: Depth=1
+; SSE42-NEXT: pmovzxbd {{.*#+}} xmm2 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; SSE42-NEXT: pmovzxbd {{.*#+}} xmm3 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero
+; SSE42-NEXT: paddd %xmm2, %xmm0
+; SSE42-NEXT: paddd %xmm3, %xmm1
+; SSE42-NEXT: addq $8, %rax
+; SSE42-NEXT: cmpq $1024, %rax # imm = 0x400
+; SSE42-NEXT: jne .LBB8_1
+; SSE42-NEXT: # %bb.2: # %middle.block
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: pshufd {{.*#+}} xmm1 = xmm0[2,3,2,3]
+; SSE42-NEXT: paddd %xmm0, %xmm1
+; SSE42-NEXT: pshufd {{.*#+}} xmm0 = xmm1[1,1,1,1]
+; SSE42-NEXT: paddd %xmm1, %xmm0
+; SSE42-NEXT: movd %xmm0, %eax
+; SSE42-NEXT: retq
+;
; AVX2-LABEL: byte_sum_v8_i32:
; AVX2: # %bb.0: # %entry
; AVX2-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX2-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX2-NEXT: xorl %eax, %eax
; AVX2-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX2-NEXT: .p2align 4
; AVX2-NEXT: .LBB8_1: # %vector.body
@@ -915,6 +1226,7 @@ define i32 @byte_sum_v8_i32() nounwind {
; AVX2-NEXT: vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
; AVX2-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX2-NEXT: addq $8, %rax
+; AVX2-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX2-NEXT: jne .LBB8_1
; AVX2-NEXT: # %bb.2: # %middle.block
; AVX2-NEXT: vextracti128 $1, %ymm0, %xmm1
@@ -930,7 +1242,7 @@ define i32 @byte_sum_v8_i32() nounwind {
; AVX512BW-LABEL: byte_sum_v8_i32:
; AVX512BW: # %bb.0: # %entry
; AVX512BW-NEXT: vpxor %xmm0, %xmm0, %xmm0
-; AVX512BW-NEXT: movq $-1024, %rax # imm = 0xFC00
+; AVX512BW-NEXT: xorl %eax, %eax
; AVX512BW-NEXT: movq a at GOTPCREL(%rip), %rcx
; AVX512BW-NEXT: .p2align 4
; AVX512BW-NEXT: .LBB8_1: # %vector.body
@@ -938,6 +1250,7 @@ define i32 @byte_sum_v8_i32() nounwind {
; AVX512BW-NEXT: vpmovzxbd {{.*#+}} ymm1 = mem[0],zero,zero,zero,mem[1],zero,zero,zero,mem[2],zero,zero,zero,mem[3],zero,zero,zero,mem[4],zero,zero,zero,mem[5],zero,zero,zero,mem[6],zero,zero,zero,mem[7],zero,zero,zero
; AVX512BW-NEXT: vpaddd %ymm0, %ymm1, %ymm0
; AVX512BW-NEXT: addq $8, %rax
+; AVX512BW-NEXT: cmpq $1024, %rax # imm = 0x400
; AVX512BW-NEXT: jne .LBB8_1
; AVX512BW-NEXT: # %bb.2: # %middle.block
; AVX512BW-NEXT: vextracti128 $1, %ymm0, %xmm1
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll
index fcd757ba163ad..0501fb4f5e063 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-debugloc.ll
@@ -1,4 +1,4 @@
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+sse2 -S | FileCheck %s
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -S | FileCheck %s
; Verify that X86PartialReduction::tryByteSumReplacement carries the
; original add instruction's !dbg location onto the freshly emitted
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll
deleted file mode 100644
index d46edfd754514..0000000000000
--- a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-negative.ll
+++ /dev/null
@@ -1,49 +0,0 @@
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+avx2 -S | FileCheck %s
-
-; Shapes that tryByteSumReplacement must not rewrite.
-
- at a = global [1024 x i8] zeroinitializer, align 16
-
-; CHECK-LABEL: @byte_sum_v8_i32
-; CHECK-NOT: psad.bw
-define i32 @byte_sum_v8_i32() nounwind {
-entry:
- br label %vector.body
-
-vector.body:
- %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
- %vec.phi = phi <8 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
- %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
- %wide.load = load <8 x i8>, ptr %p, align 8
- %z = zext <8 x i8> %wide.load to <8 x i32>
- %add = add nsw <8 x i32> %z, %vec.phi
- %index.next = add i64 %index, 8
- %cmp = icmp eq i64 %index.next, 1024
- br i1 %cmp, label %middle.block, label %vector.body
-
-middle.block:
- %ext = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %add)
- ret i32 %ext
-}
-
-; CHECK-LABEL: @byte_sum_v24_i32
-; CHECK-NOT: psad.bw
-define i32 @byte_sum_v24_i32() nounwind {
-entry:
- br label %vector.body
-
-vector.body:
- %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
- %vec.phi = phi <24 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
- %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
- %wide.load = load <24 x i8>, ptr %p, align 8
- %z = zext <24 x i8> %wide.load to <24 x i32>
- %add = add nsw <24 x i32> %z, %vec.phi
- %index.next = add i64 %index, 24
- %cmp = icmp eq i64 %index.next, 1024
- br i1 %cmp, label %middle.block, label %vector.body
-
-middle.block:
- %ext = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %add)
- ret i32 %ext
-}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll
deleted file mode 100644
index 23be1178bf767..0000000000000
--- a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum-wide-i64.ll
+++ /dev/null
@@ -1,59 +0,0 @@
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+avx2 -S | FileCheck %s --check-prefix=AVX2
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+avx512bw -S | FileCheck %s --check-prefix=AVX512
-
-; Wider-VF i64-accumulator positive shapes for tryByteSumReplacement.
-; These exercise the PerSplitTy = <IntrinsicNumElts/8 x i64> branch where the
-; matcher must NOT bitcast the psadbw result back to i32.
-;
-; VF=32 i64 uses the AVX2 256-bit lane (one avx2.psad.bw call). On
-; +avx512bw the dispatch falls into the same AVX2 path because NumElts<64.
-;
-; VF=64 i64 uses the AVX-512BW 512-bit lane (one avx512.psad.bw.512 call).
-
- at a = global [1024 x i8] zeroinitializer, align 16
-
-; AVX2-LABEL: @byte_sum_v32_i64(
-; AVX2: call <4 x i64> @llvm.x86.avx2.psad.bw(
-; AVX512-LABEL: @byte_sum_v32_i64(
-; AVX512: call <4 x i64> @llvm.x86.avx2.psad.bw(
-define i64 @byte_sum_v32_i64() nounwind {
-entry:
- br label %vector.body
-
-vector.body:
- %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
- %vec.phi = phi <32 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
- %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
- %wide.load = load <32 x i8>, ptr %p, align 16
- %z = zext <32 x i8> %wide.load to <32 x i64>
- %add = add nsw <32 x i64> %z, %vec.phi
- %index.next = add i64 %index, 32
- %cmp = icmp eq i64 %index.next, 1024
- br i1 %cmp, label %middle.block, label %vector.body
-
-middle.block:
- %ext = call i64 @llvm.vector.reduce.add.v32i64(<32 x i64> %add)
- ret i64 %ext
-}
-
-; AVX512-LABEL: @byte_sum_v64_i64(
-; AVX512: call <8 x i64> @llvm.x86.avx512.psad.bw.512(
-define i64 @byte_sum_v64_i64() nounwind {
-entry:
- br label %vector.body
-
-vector.body:
- %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
- %vec.phi = phi <64 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
- %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
- %wide.load = load <64 x i8>, ptr %p, align 16
- %z = zext <64 x i8> %wide.load to <64 x i64>
- %add = add nsw <64 x i64> %z, %vec.phi
- %index.next = add i64 %index, 64
- %cmp = icmp eq i64 %index.next, 1024
- br i1 %cmp, label %middle.block, label %vector.body
-
-middle.block:
- %ext = call i64 @llvm.vector.reduce.add.v64i64(<64 x i64> %add)
- ret i64 %ext
-}
diff --git a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
index 6acd8de044aa2..9653ae7afd249 100644
--- a/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
+++ b/llvm/test/Transforms/PhaseOrdering/X86/x86-partial-reduction-byte-sum.ll
@@ -1,9 +1,15 @@
-; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mattr=+sse2 -S | FileCheck %s
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64 -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v2 -S | FileCheck %s --check-prefixes=CHECK,SSE
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v3 -S | FileCheck %s --check-prefixes=CHECK,AVX2
+; RUN: opt < %s -passes='expand-reductions,x86-partial-reduction' -mtriple=x86_64-unknown-unknown -mcpu=x86-64-v4 -S | FileCheck %s --check-prefixes=CHECK,AVX512
-; Isolate X86PartialReduction::tryByteSumReplacement on a positive shape.
+; Test X86PartialReduction::tryByteSumReplacement: positive and negative shapes.
@a = global [1024 x i8] zeroinitializer, align 16
+;; Positive cases -----------------------------------------------------------
+
+; CHECK-LABEL: @byte_sum_v16_i32(
; CHECK: call <2 x i64> @llvm.x86.sse2.psad.bw(
define i32 @byte_sum_v16_i32() nounwind {
entry:
@@ -24,3 +30,103 @@ middle.block:
%ext = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %add)
ret i32 %ext
}
+
+; SSE-LABEL: @byte_sum_v32_i64(
+; SSE: call <2 x i64> @llvm.x86.sse2.psad.bw(
+; AVX2-LABEL: @byte_sum_v32_i64(
+; AVX2: call <4 x i64> @llvm.x86.avx2.psad.bw(
+; AVX512-LABEL: @byte_sum_v32_i64(
+; AVX512: call <4 x i64> @llvm.x86.avx2.psad.bw(
+define i64 @byte_sum_v32_i64() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <32 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <32 x i8>, ptr %p, align 16
+ %z = zext <32 x i8> %wide.load to <32 x i64>
+ %add = add nsw <32 x i64> %z, %vec.phi
+ %index.next = add i64 %index, 32
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i64 @llvm.vector.reduce.add.v32i64(<32 x i64> %add)
+ ret i64 %ext
+}
+
+; SSE-LABEL: @byte_sum_v64_i64(
+; SSE: call <2 x i64> @llvm.x86.sse2.psad.bw(
+; AVX2-LABEL: @byte_sum_v64_i64(
+; AVX2: call <4 x i64> @llvm.x86.avx2.psad.bw(
+; AVX512-LABEL: @byte_sum_v64_i64(
+; AVX512: call <8 x i64> @llvm.x86.avx512.psad.bw.512(
+define i64 @byte_sum_v64_i64() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <64 x i64> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <64 x i8>, ptr %p, align 16
+ %z = zext <64 x i8> %wide.load to <64 x i64>
+ %add = add nsw <64 x i64> %z, %vec.phi
+ %index.next = add i64 %index, 64
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i64 @llvm.vector.reduce.add.v64i64(<64 x i64> %add)
+ ret i64 %ext
+}
+
+;; Negative cases -----------------------------------------------------------
+
+; CHECK-LABEL: @byte_sum_v8_i32(
+; CHECK-NOT: psad.bw
+; CHECK: ret i32
+define i32 @byte_sum_v8_i32() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <8 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <8 x i8>, ptr %p, align 8
+ %z = zext <8 x i8> %wide.load to <8 x i32>
+ %add = add nsw <8 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 8
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %add)
+ ret i32 %ext
+}
+
+; CHECK-LABEL: @byte_sum_v24_i32(
+; CHECK-NOT: psad.bw
+; CHECK: ret i32
+define i32 @byte_sum_v24_i32() nounwind {
+entry:
+ br label %vector.body
+
+vector.body:
+ %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+ %vec.phi = phi <24 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+ %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+ %wide.load = load <24 x i8>, ptr %p, align 8
+ %z = zext <24 x i8> %wide.load to <24 x i32>
+ %add = add nsw <24 x i32> %z, %vec.phi
+ %index.next = add i64 %index, 24
+ %cmp = icmp eq i64 %index.next, 1024
+ br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+ %ext = call i32 @llvm.vector.reduce.add.v24i32(<24 x i32> %add)
+ ret i32 %ext
+}
More information about the llvm-commits
mailing list