[llvm] [X86][PartialReduction] Lower zext-byte add reductions to vpsadbw (PR #201076)

[Simon Pilgrim via llvm-commits]

================
@@ -0,0 +1,206 @@
+; RUN: llc < %s -mtriple=x86_64-unknown-unknown -mattr=+avx2 | FileCheck %s --check-prefix=NO-PSADBW
+
+; Negative CodeGen tests: must NOT lower to vpsadbw via tryByteSumReplacement.
+
+ at a = dso_local global [1024 x i8] zeroinitializer, align 16
+
+define dso_local i32 @byte_sum_v16_i32_sext() nounwind {
+; NO-PSADBW-LABEL: byte_sum_v16_i32_sext:
+entry:
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+  %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+  %wide.load = load <16 x i8>, ptr %p, align 16
+  %z = sext <16 x i8> %wide.load to <16 x i32>
+  %add = add nsw <16 x i32> %z, %vec.phi
+  %index.next = add i64 %index, 16
+  %cmp = icmp eq i64 %index.next, 1024
+  br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+  %s1 = shufflevector <16 x i32> %add, <16 x i32> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r1 = add <16 x i32> %add, %s1
+  %s2 = shufflevector <16 x i32> %r1, <16 x i32> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r2 = add <16 x i32> %r1, %s2
+  %s3 = shufflevector <16 x i32> %r2, <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r3 = add <16 x i32> %r2, %s3
+  %s4 = shufflevector <16 x i32> %r3, <16 x i32> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r4 = add <16 x i32> %r3, %s4
+  %ext = extractelement <16 x i32> %r4, i32 0
+  ret i32 %ext
+; NO-PSADBW-NOT: psadbw
+}
+
+;----------------------------------------------------------------------------
+; NEGATIVE: i16 source instead of i8.
+;
+; psadbw operates on bytes; widening from i16 is the wrong shape so the pass
+; must bail.  Nothing here looks like a SAD/MAdd either, so SDAG also will
+; not invent a psadbw.
+;----------------------------------------------------------------------------
+define dso_local i32 @word_sum_v16_i32(ptr nocapture readonly %src) nounwind {
+; NO-PSADBW-LABEL: word_sum_v16_i32:
+entry:
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+  %p = getelementptr inbounds i16, ptr %src, i64 %index
+  %wide.load = load <16 x i16>, ptr %p, align 2
+  %z = zext <16 x i16> %wide.load to <16 x i32>
+  %add = add nsw <16 x i32> %z, %vec.phi
+  %index.next = add i64 %index, 16
+  %cmp = icmp eq i64 %index.next, 1024
+  br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+  %s1 = shufflevector <16 x i32> %add, <16 x i32> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r1 = add <16 x i32> %add, %s1
+  %s2 = shufflevector <16 x i32> %r1, <16 x i32> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r2 = add <16 x i32> %r1, %s2
+  %s3 = shufflevector <16 x i32> %r2, <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r3 = add <16 x i32> %r2, %s3
+  %s4 = shufflevector <16 x i32> %r3, <16 x i32> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r4 = add <16 x i32> %r3, %s4
+  %ext = extractelement <16 x i32> %r4, i32 0
+  ret i32 %ext
+; NO-PSADBW-NOT: psadbw
+}
+
+;----------------------------------------------------------------------------
+; NEGATIVE: i16 accumulator (element type < 32 bits).
+;
+; psadbw produces i64 partial sums; the rewrite needs at least i32 lanes to
+; hold them.  An i16 reduction must not be rewritten.
+;----------------------------------------------------------------------------
+define dso_local i16 @byte_sum_v16_i16() nounwind {
+; NO-PSADBW-LABEL: byte_sum_v16_i16:
+entry:
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %vec.phi = phi <16 x i16> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+  %p = getelementptr inbounds [1024 x i8], ptr @a, i64 0, i64 %index
+  %wide.load = load <16 x i8>, ptr %p, align 16
+  %z = zext <16 x i8> %wide.load to <16 x i16>
+  %add = add nsw <16 x i16> %z, %vec.phi
+  %index.next = add i64 %index, 16
+  %cmp = icmp eq i64 %index.next, 1024
+  br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+  %s1 = shufflevector <16 x i16> %add, <16 x i16> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r1 = add <16 x i16> %add, %s1
+  %s2 = shufflevector <16 x i16> %r1, <16 x i16> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r2 = add <16 x i16> %r1, %s2
+  %s3 = shufflevector <16 x i16> %r2, <16 x i16> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r3 = add <16 x i16> %r2, %s3
+  %s4 = shufflevector <16 x i16> %r3, <16 x i16> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r4 = add <16 x i16> %r3, %s4
+  %ext = extractelement <16 x i16> %r4, i32 0
+  ret i16 %ext
+; NO-PSADBW-NOT: psadbw
+}
+
+;----------------------------------------------------------------------------
+; NEGATIVE: leaf is not a zext.
+;
+; The leaf operand of the reduction is an already-i32 value (no zext, no
+; underlying i8 source).  tryByteSumReplacement must bail because there is
+; no way to know the values fit in a byte.
+;----------------------------------------------------------------------------
+define dso_local i32 @dword_sum_v16_i32(ptr nocapture readonly %src) nounwind {
+; NO-PSADBW-LABEL: dword_sum_v16_i32:
+entry:
+  br label %vector.body
+
+vector.body:
+  %index = phi i64 [ 0, %entry ], [ %index.next, %vector.body ]
+  %vec.phi = phi <16 x i32> [ zeroinitializer, %entry ], [ %add, %vector.body ]
+  %p = getelementptr inbounds i32, ptr %src, i64 %index
+  %wide.load = load <16 x i32>, ptr %p, align 4
+  %add = add nsw <16 x i32> %wide.load, %vec.phi
+  %index.next = add i64 %index, 16
+  %cmp = icmp eq i64 %index.next, 1024
+  br i1 %cmp, label %middle.block, label %vector.body
+
+middle.block:
+  %s1 = shufflevector <16 x i32> %add, <16 x i32> poison, <16 x i32> <i32 8, i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r1 = add <16 x i32> %add, %s1
+  %s2 = shufflevector <16 x i32> %r1, <16 x i32> poison, <16 x i32> <i32 4, i32 5, i32 6, i32 7, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r2 = add <16 x i32> %r1, %s2
+  %s3 = shufflevector <16 x i32> %r2, <16 x i32> poison, <16 x i32> <i32 2, i32 3, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r3 = add <16 x i32> %r2, %s3
+  %s4 = shufflevector <16 x i32> %r3, <16 x i32> poison, <16 x i32> <i32 1, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison, i32 poison>
+  %r4 = add <16 x i32> %r3, %s4
+  %ext = extractelement <16 x i32> %r4, i32 0
----------------
RKSimon wrote:

use a vector.reduce.add.v16i32 call here instead of the expanded codegen

https://github.com/llvm/llvm-project/pull/201076