[llvm] c415ff1 - [AArch64] Add extra vecreduce.add tests, including extending reductions. NFC
David Green via llvm-commits
llvm-commits at lists.llvm.org
Tue Jan 25 10:10:15 PST 2022
Author: David Green
Date: 2022-01-25T18:10:09Z
New Revision: c415ff186dbb9891886c6ac1f03060ad537e7074
URL: https://github.com/llvm/llvm-project/commit/c415ff186dbb9891886c6ac1f03060ad537e7074
DIFF: https://github.com/llvm/llvm-project/commit/c415ff186dbb9891886c6ac1f03060ad537e7074.diff
LOG: [AArch64] Add extra vecreduce.add tests, including extending reductions. NFC
This is all the reductions from i8 -> i64 with either sign or zero
extensions.
Added:
llvm/test/CodeGen/AArch64/vecreduce-add.ll
Modified:
Removed:
################################################################################
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add.ll b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
new file mode 100644
index 0000000000000..06077071468e2
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vecreduce-add.ll
@@ -0,0 +1,2095 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-BASE
+; RUN: llc -mtriple=aarch64-none-linux-gnu -mattr=+dotprod %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-DOT
+
+define i32 @add_v4i32_v4i32(<4 x i32> %x) {
+; CHECK-LABEL: add_v4i32_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x)
+ ret i32 %z
+}
+
+define i64 @add_v4i32_v4i64_zext(<4 x i32> %x) {
+; CHECK-LABEL: add_v4i32_v4i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i32> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v4i32_v4i64_sext(<4 x i32> %x) {
+; CHECK-LABEL: add_v4i32_v4i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-NEXT: saddw2 v0.2d, v1.2d, v0.4s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i32> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v2i32_v2i64_zext(<2 x i32> %x) {
+; CHECK-LABEL: add_v2i32_v2i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <2 x i32> %x to <2 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v2i32_v2i64_sext(<2 x i32> %x) {
+; CHECK-LABEL: add_v2i32_v2i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <2 x i32> %x to <2 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ ret i64 %z
+}
+
+define i32 @add_v8i16_v8i32_zext(<8 x i16> %x) {
+; CHECK-LABEL: add_v8i16_v8i32_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-NEXT: uaddw2 v0.4s, v1.4s, v0.8h
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <8 x i16> %x to <8 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ ret i32 %z
+}
+
+define i32 @add_v8i16_v8i32_sext(<8 x i16> %x) {
+; CHECK-LABEL: add_v8i16_v8i32_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-NEXT: saddw2 v0.4s, v1.4s, v0.8h
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <8 x i16> %x to <8 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ ret i32 %z
+}
+
+define i32 @add_v4i16_v4i32_zext(<4 x i16> %x) {
+; CHECK-LABEL: add_v4i16_v4i32_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i16> %x to <4 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ ret i32 %z
+}
+
+define i32 @add_v4i16_v4i32_sext(<4 x i16> %x) {
+; CHECK-LABEL: add_v4i16_v4i32_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i16> %x to <4 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ ret i32 %z
+}
+
+define zeroext i16 @add_v8i16_v8i16(<8 x i16> %x) {
+; CHECK-LABEL: add_v8i16_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %x)
+ ret i16 %z
+}
+
+define i64 @add_v8i16_v8i64_zext(<8 x i16> %x) {
+; CHECK-LABEL: add_v8i16_v8i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: uaddl2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <8 x i16> %x to <8 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v8i16_v8i64_sext(<8 x i16> %x) {
+; CHECK-LABEL: add_v8i16_v8i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: saddl2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: saddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <8 x i16> %x to <8 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v4i16_v4i64_zext(<4 x i16> %x) {
+; CHECK-LABEL: add_v4i16_v4i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i16> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v4i16_v4i64_sext(<4 x i16> %x) {
+; CHECK-LABEL: add_v4i16_v4i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-NEXT: saddw2 v0.2d, v1.2d, v0.4s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i16> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v2i16_v2i64_zext(<2 x i16> %x) {
+; CHECK-LABEL: add_v2i16_v2i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d1, #0x00ffff0000ffff
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <2 x i16> %x to <2 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v2i16_v2i64_sext(<2 x i16> %x) {
+; CHECK-LABEL: add_v2i16_v2i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: shl v0.2d, v0.2d, #48
+; CHECK-NEXT: sshr v0.2d, v0.2d, #48
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <2 x i16> %x to <2 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ ret i64 %z
+}
+
+define i32 @add_v16i8_v16i32_zext(<16 x i8> %x) {
+; CHECK-BASE-LABEL: add_v16i8_v16i32_zext:
+; CHECK-BASE: // %bb.0: // %entry
+; CHECK-BASE-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BASE-NEXT: uaddl2 v2.4s, v0.8h, v1.8h
+; CHECK-BASE-NEXT: uaddl v0.4s, v0.4h, v1.4h
+; CHECK-BASE-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-BASE-NEXT: addv s0, v0.4s
+; CHECK-BASE-NEXT: fmov w0, s0
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-DOT-LABEL: add_v16i8_v16i32_zext:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v1.16b, #1
+; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-DOT-NEXT: udot v2.4s, v0.16b, v1.16b
+; CHECK-DOT-NEXT: addv s0, v2.4s
+; CHECK-DOT-NEXT: fmov w0, s0
+; CHECK-DOT-NEXT: ret
+entry:
+ %xx = zext <16 x i8> %x to <16 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx)
+ ret i32 %z
+}
+
+define i32 @add_v16i8_v16i32_sext(<16 x i8> %x) {
+; CHECK-BASE-LABEL: add_v16i8_v16i32_sext:
+; CHECK-BASE: // %bb.0: // %entry
+; CHECK-BASE-NEXT: sshll2 v1.8h, v0.16b, #0
+; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-BASE-NEXT: saddl2 v2.4s, v0.8h, v1.8h
+; CHECK-BASE-NEXT: saddl v0.4s, v0.4h, v1.4h
+; CHECK-BASE-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-BASE-NEXT: addv s0, v0.4s
+; CHECK-BASE-NEXT: fmov w0, s0
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-DOT-LABEL: add_v16i8_v16i32_sext:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v1.16b, #1
+; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b
+; CHECK-DOT-NEXT: addv s0, v2.4s
+; CHECK-DOT-NEXT: fmov w0, s0
+; CHECK-DOT-NEXT: ret
+entry:
+ %xx = sext <16 x i8> %x to <16 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx)
+ ret i32 %z
+}
+
+define i32 @add_v8i8_v8i32_zext(<8 x i8> %x) {
+; CHECK-BASE-LABEL: add_v8i8_v8i32_zext:
+; CHECK-BASE: // %bb.0: // %entry
+; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BASE-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-BASE-NEXT: uaddw2 v0.4s, v1.4s, v0.8h
+; CHECK-BASE-NEXT: addv s0, v0.4s
+; CHECK-BASE-NEXT: fmov w0, s0
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-DOT-LABEL: add_v8i8_v8i32_zext:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v1.8b, #1
+; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-DOT-NEXT: udot v2.2s, v0.8b, v1.8b
+; CHECK-DOT-NEXT: addp v0.2s, v2.2s, v2.2s
+; CHECK-DOT-NEXT: fmov w0, s0
+; CHECK-DOT-NEXT: ret
+entry:
+ %xx = zext <8 x i8> %x to <8 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ ret i32 %z
+}
+
+define i32 @add_v8i8_v8i32_sext(<8 x i8> %x) {
+; CHECK-BASE-LABEL: add_v8i8_v8i32_sext:
+; CHECK-BASE: // %bb.0: // %entry
+; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-BASE-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-BASE-NEXT: saddw2 v0.4s, v1.4s, v0.8h
+; CHECK-BASE-NEXT: addv s0, v0.4s
+; CHECK-BASE-NEXT: fmov w0, s0
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-DOT-LABEL: add_v8i8_v8i32_sext:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v1.8b, #1
+; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-DOT-NEXT: sdot v2.2s, v0.8b, v1.8b
+; CHECK-DOT-NEXT: addp v0.2s, v2.2s, v2.2s
+; CHECK-DOT-NEXT: fmov w0, s0
+; CHECK-DOT-NEXT: ret
+entry:
+ %xx = sext <8 x i8> %x to <8 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ ret i32 %z
+}
+
+define i32 @add_v4i8_v4i32_zext(<4 x i8> %x) {
+; CHECK-LABEL: add_v4i8_v4i32_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i8> %x to <4 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ ret i32 %z
+}
+
+define i32 @add_v4i8_v4i32_sext(<4 x i8> %x) {
+; CHECK-LABEL: add_v4i8_v4i32_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: shl v0.4s, v0.4s, #24
+; CHECK-NEXT: sshr v0.4s, v0.4s, #24
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i8> %x to <4 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ ret i32 %z
+}
+
+define zeroext i16 @add_v16i8_v16i16_zext(<16 x i8> %x) {
+; CHECK-LABEL: add_v16i8_v16i16_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-NEXT: uaddw2 v0.8h, v1.8h, v0.16b
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <16 x i8> %x to <16 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)
+ ret i16 %z
+}
+
+define signext i16 @add_v16i8_v16i16_sext(<16 x i8> %x) {
+; CHECK-LABEL: add_v16i8_v16i16_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v1.8h, v0.8b, #0
+; CHECK-NEXT: saddw2 v0.8h, v1.8h, v0.16b
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: smov w0, v0.h[0]
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <16 x i8> %x to <16 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)
+ ret i16 %z
+}
+
+define zeroext i16 @add_v8i8_v8i16_zext(<8 x i8> %x) {
+; CHECK-LABEL: add_v8i8_v8i16_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <8 x i8> %x to <8 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)
+ ret i16 %z
+}
+
+define signext i16 @add_v8i8_v8i16_sext(<8 x i8> %x) {
+; CHECK-LABEL: add_v8i8_v8i16_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: smov w0, v0.h[0]
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <8 x i8> %x to <8 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)
+ ret i16 %z
+}
+
+define zeroext i8 @add_v16i8_v16i8(<16 x i8> %x) {
+; CHECK-LABEL: add_v16i8_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: addv b0, v0.16b
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %x)
+ ret i8 %z
+}
+
+define i64 @add_v16i8_v16i64_zext(<16 x i8> %x) {
+; CHECK-LABEL: add_v16i8_v16i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v2.4s, v1.8h, #0
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: uaddl2 v4.2d, v3.4s, v2.4s
+; CHECK-NEXT: uaddl2 v5.2d, v0.4s, v1.4s
+; CHECK-NEXT: uaddl v2.2d, v3.2s, v2.2s
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v1.2d, v5.2d, v4.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <16 x i8> %x to <16 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v16i8_v16i64_sext(<16 x i8> %x) {
+; CHECK-LABEL: add_v16i8_v16i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll2 v1.8h, v0.16b, #0
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: sshll2 v2.4s, v1.8h, #0
+; CHECK-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-NEXT: sshll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: saddl2 v4.2d, v3.4s, v2.4s
+; CHECK-NEXT: saddl2 v5.2d, v0.4s, v1.4s
+; CHECK-NEXT: saddl v2.2d, v3.2s, v2.2s
+; CHECK-NEXT: saddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v1.2d, v5.2d, v4.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <16 x i8> %x to <16 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v8i8_v8i64_zext(<8 x i8> %x) {
+; CHECK-LABEL: add_v8i8_v8i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: uaddl2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <8 x i8> %x to <8 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v8i8_v8i64_sext(<8 x i8> %x) {
+; CHECK-LABEL: add_v8i8_v8i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: saddl2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: saddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <8 x i8> %x to <8 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v4i8_v4i64_zext(<4 x i8> %x) {
+; CHECK-LABEL: add_v4i8_v4i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i8> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v4i8_v4i64_sext(<4 x i8> %x) {
+; CHECK-LABEL: add_v4i8_v4i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NEXT: shl v1.2d, v1.2d, #56
+; CHECK-NEXT: shl v0.2d, v0.2d, #56
+; CHECK-NEXT: sshr v1.2d, v1.2d, #56
+; CHECK-NEXT: ssra v1.2d, v0.2d, #56
+; CHECK-NEXT: addp d0, v1.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i8> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v2i8_v2i64_zext(<2 x i8> %x) {
+; CHECK-LABEL: add_v2i8_v2i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d1, #0x0000ff000000ff
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <2 x i8> %x to <2 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v2i8_v2i64_sext(<2 x i8> %x) {
+; CHECK-LABEL: add_v2i8_v2i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: shl v0.2d, v0.2d, #56
+; CHECK-NEXT: sshr v0.2d, v0.2d, #56
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <2 x i8> %x to <2 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ ret i64 %z
+}
+
+define i64 @add_v2i64_v2i64(<2 x i64> %x) {
+; CHECK-LABEL: add_v2i64_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %x)
+ ret i64 %z
+}
+
+define i32 @add_v4i32_v4i32_acc(<4 x i32> %x, i32 %a) {
+; CHECK-LABEL: add_v4i32_v4i32_acc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w0
+; CHECK-NEXT: ret
+entry:
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x)
+ %r = add i32 %z, %a
+ ret i32 %r
+}
+
+define i64 @add_v4i32_v4i64_acc_zext(<4 x i32> %x, i64 %a) {
+; CHECK-LABEL: add_v4i32_v4i64_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i32> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v4i32_v4i64_acc_sext(<4 x i32> %x, i64 %a) {
+; CHECK-LABEL: add_v4i32_v4i64_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-NEXT: saddw2 v0.2d, v1.2d, v0.4s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i32> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v2i32_v2i64_acc_zext(<2 x i32> %x, i64 %a) {
+; CHECK-LABEL: add_v2i32_v2i64_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <2 x i32> %x to <2 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v2i32_v2i64_acc_sext(<2 x i32> %x, i64 %a) {
+; CHECK-LABEL: add_v2i32_v2i64_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.2d, v0.2s, #0
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <2 x i32> %x to <2 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i32 @add_v8i16_v8i32_acc_zext(<8 x i16> %x, i32 %a) {
+; CHECK-LABEL: add_v8i16_v8i32_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-NEXT: uaddw2 v0.4s, v1.4s, v0.8h
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <8 x i16> %x to <8 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ %r = add i32 %z, %a
+ ret i32 %r
+}
+
+define i32 @add_v8i16_v8i32_acc_sext(<8 x i16> %x, i32 %a) {
+; CHECK-LABEL: add_v8i16_v8i32_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-NEXT: saddw2 v0.4s, v1.4s, v0.8h
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <8 x i16> %x to <8 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ %r = add i32 %z, %a
+ ret i32 %r
+}
+
+define i32 @add_v4i16_v4i32_acc_zext(<4 x i16> %x, i32 %a) {
+; CHECK-LABEL: add_v4i16_v4i32_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i16> %x to <4 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ %r = add i32 %z, %a
+ ret i32 %r
+}
+
+define i32 @add_v4i16_v4i32_acc_sext(<4 x i16> %x, i32 %a) {
+; CHECK-LABEL: add_v4i16_v4i32_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i16> %x to <4 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ %r = add i32 %z, %a
+ ret i32 %r
+}
+
+define zeroext i16 @add_v8i16_v8i16_acc(<8 x i16> %x, i16 %a) {
+; CHECK-LABEL: add_v8i16_v8i16_acc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w8, w8, w0
+; CHECK-NEXT: and w0, w8, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %x)
+ %r = add i16 %z, %a
+ ret i16 %r
+}
+
+define i64 @add_v8i16_v8i64_acc_zext(<8 x i16> %x, i64 %a) {
+; CHECK-LABEL: add_v8i16_v8i64_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: uaddl2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <8 x i16> %x to <8 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v8i16_v8i64_acc_sext(<8 x i16> %x, i64 %a) {
+; CHECK-LABEL: add_v8i16_v8i64_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: saddl2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: saddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <8 x i16> %x to <8 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v4i16_v4i64_acc_zext(<4 x i16> %x, i64 %a) {
+; CHECK-LABEL: add_v4i16_v4i64_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i16> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v4i16_v4i64_acc_sext(<4 x i16> %x, i64 %a) {
+; CHECK-LABEL: add_v4i16_v4i64_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: sshll v1.2d, v0.2s, #0
+; CHECK-NEXT: saddw2 v0.2d, v1.2d, v0.4s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i16> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v2i16_v2i64_acc_zext(<2 x i16> %x, i64 %a) {
+; CHECK-LABEL: add_v2i16_v2i64_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d1, #0x00ffff0000ffff
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <2 x i16> %x to <2 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v2i16_v2i64_acc_sext(<2 x i16> %x, i64 %a) {
+; CHECK-LABEL: add_v2i16_v2i64_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: shl v0.2d, v0.2d, #48
+; CHECK-NEXT: sshr v0.2d, v0.2d, #48
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <2 x i16> %x to <2 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i32 @add_v16i8_v16i32_acc_zext(<16 x i8> %x, i32 %a) {
+; CHECK-BASE-LABEL: add_v16i8_v16i32_acc_zext:
+; CHECK-BASE: // %bb.0: // %entry
+; CHECK-BASE-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BASE-NEXT: uaddl2 v2.4s, v0.8h, v1.8h
+; CHECK-BASE-NEXT: uaddl v0.4s, v0.4h, v1.4h
+; CHECK-BASE-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-BASE-NEXT: addv s0, v0.4s
+; CHECK-BASE-NEXT: fmov w8, s0
+; CHECK-BASE-NEXT: add w0, w8, w0
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-DOT-LABEL: add_v16i8_v16i32_acc_zext:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v1.16b, #1
+; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-DOT-NEXT: udot v2.4s, v0.16b, v1.16b
+; CHECK-DOT-NEXT: addv s0, v2.4s
+; CHECK-DOT-NEXT: fmov w8, s0
+; CHECK-DOT-NEXT: add w0, w8, w0
+; CHECK-DOT-NEXT: ret
+entry:
+ %xx = zext <16 x i8> %x to <16 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx)
+ %r = add i32 %z, %a
+ ret i32 %r
+}
+
+define i32 @add_v16i8_v16i32_acc_sext(<16 x i8> %x, i32 %a) {
+; CHECK-BASE-LABEL: add_v16i8_v16i32_acc_sext:
+; CHECK-BASE: // %bb.0: // %entry
+; CHECK-BASE-NEXT: sshll2 v1.8h, v0.16b, #0
+; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-BASE-NEXT: saddl2 v2.4s, v0.8h, v1.8h
+; CHECK-BASE-NEXT: saddl v0.4s, v0.4h, v1.4h
+; CHECK-BASE-NEXT: add v0.4s, v0.4s, v2.4s
+; CHECK-BASE-NEXT: addv s0, v0.4s
+; CHECK-BASE-NEXT: fmov w8, s0
+; CHECK-BASE-NEXT: add w0, w8, w0
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-DOT-LABEL: add_v16i8_v16i32_acc_sext:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v1.16b, #1
+; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-DOT-NEXT: sdot v2.4s, v0.16b, v1.16b
+; CHECK-DOT-NEXT: addv s0, v2.4s
+; CHECK-DOT-NEXT: fmov w8, s0
+; CHECK-DOT-NEXT: add w0, w8, w0
+; CHECK-DOT-NEXT: ret
+entry:
+ %xx = sext <16 x i8> %x to <16 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx)
+ %r = add i32 %z, %a
+ ret i32 %r
+}
+
+define i32 @add_v8i8_v8i32_acc_zext(<8 x i8> %x, i32 %a) {
+; CHECK-BASE-LABEL: add_v8i8_v8i32_acc_zext:
+; CHECK-BASE: // %bb.0: // %entry
+; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BASE-NEXT: ushll v1.4s, v0.4h, #0
+; CHECK-BASE-NEXT: uaddw2 v0.4s, v1.4s, v0.8h
+; CHECK-BASE-NEXT: addv s0, v0.4s
+; CHECK-BASE-NEXT: fmov w8, s0
+; CHECK-BASE-NEXT: add w0, w8, w0
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-DOT-LABEL: add_v8i8_v8i32_acc_zext:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v1.8b, #1
+; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-DOT-NEXT: udot v2.2s, v0.8b, v1.8b
+; CHECK-DOT-NEXT: addp v0.2s, v2.2s, v2.2s
+; CHECK-DOT-NEXT: fmov w8, s0
+; CHECK-DOT-NEXT: add w0, w8, w0
+; CHECK-DOT-NEXT: ret
+entry:
+ %xx = zext <8 x i8> %x to <8 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ %r = add i32 %z, %a
+ ret i32 %r
+}
+
+define i32 @add_v8i8_v8i32_acc_sext(<8 x i8> %x, i32 %a) {
+; CHECK-BASE-LABEL: add_v8i8_v8i32_acc_sext:
+; CHECK-BASE: // %bb.0: // %entry
+; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-BASE-NEXT: sshll v1.4s, v0.4h, #0
+; CHECK-BASE-NEXT: saddw2 v0.4s, v1.4s, v0.8h
+; CHECK-BASE-NEXT: addv s0, v0.4s
+; CHECK-BASE-NEXT: fmov w8, s0
+; CHECK-BASE-NEXT: add w0, w8, w0
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-DOT-LABEL: add_v8i8_v8i32_acc_sext:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v1.8b, #1
+; CHECK-DOT-NEXT: movi v2.2d, #0000000000000000
+; CHECK-DOT-NEXT: sdot v2.2s, v0.8b, v1.8b
+; CHECK-DOT-NEXT: addp v0.2s, v2.2s, v2.2s
+; CHECK-DOT-NEXT: fmov w8, s0
+; CHECK-DOT-NEXT: add w0, w8, w0
+; CHECK-DOT-NEXT: ret
+entry:
+ %xx = sext <8 x i8> %x to <8 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ %r = add i32 %z, %a
+ ret i32 %r
+}
+
+define i32 @add_v4i8_v4i32_acc_zext(<4 x i8> %x, i32 %a) {
+; CHECK-LABEL: add_v4i8_v4i32_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i8> %x to <4 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ %r = add i32 %z, %a
+ ret i32 %r
+}
+
+define i32 @add_v4i8_v4i32_acc_sext(<4 x i8> %x, i32 %a) {
+; CHECK-LABEL: add_v4i8_v4i32_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: shl v0.4s, v0.4s, #24
+; CHECK-NEXT: sshr v0.4s, v0.4s, #24
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w0, w8, w0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i8> %x to <4 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ %r = add i32 %z, %a
+ ret i32 %r
+}
+
+define zeroext i16 @add_v16i8_v16i16_acc_zext(<16 x i8> %x, i16 %a) {
+; CHECK-LABEL: add_v16i8_v16i16_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v1.8h, v0.8b, #0
+; CHECK-NEXT: uaddw2 v0.8h, v1.8h, v0.16b
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w8, w8, w0
+; CHECK-NEXT: and w0, w8, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <16 x i8> %x to <16 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)
+ %r = add i16 %z, %a
+ ret i16 %r
+}
+
+define signext i16 @add_v16i8_v16i16_acc_sext(<16 x i8> %x, i16 %a) {
+; CHECK-LABEL: add_v16i8_v16i16_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v1.8h, v0.8b, #0
+; CHECK-NEXT: saddw2 v0.8h, v1.8h, v0.16b
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w8, w8, w0
+; CHECK-NEXT: sxth w0, w8
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <16 x i8> %x to <16 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)
+ %r = add i16 %z, %a
+ ret i16 %r
+}
+
+define zeroext i16 @add_v8i8_v8i16_acc_zext(<8 x i8> %x, i16 %a) {
+; CHECK-LABEL: add_v8i8_v8i16_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w8, w8, w0
+; CHECK-NEXT: and w0, w8, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <8 x i8> %x to <8 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)
+ %r = add i16 %z, %a
+ ret i16 %r
+}
+
+define signext i16 @add_v8i8_v8i16_acc_sext(<8 x i8> %x, i16 %a) {
+; CHECK-LABEL: add_v8i8_v8i16_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w8, w8, w0
+; CHECK-NEXT: sxth w0, w8
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <8 x i8> %x to <8 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)
+ %r = add i16 %z, %a
+ ret i16 %r
+}
+
+define zeroext i8 @add_v16i8_v16i8_acc(<16 x i8> %x, i8 %a) {
+; CHECK-LABEL: add_v16i8_v16i8_acc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: addv b0, v0.16b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: add w8, w8, w0
+; CHECK-NEXT: and w0, w8, #0xff
+; CHECK-NEXT: ret
+entry:
+ %z = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %x)
+ %r = add i8 %z, %a
+ ret i8 %r
+}
+
+define i64 @add_v16i8_v16i64_acc_zext(<16 x i8> %x, i64 %a) {
+; CHECK-LABEL: add_v16i8_v16i64_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll2 v1.8h, v0.16b, #0
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v2.4s, v1.8h, #0
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: uaddl2 v4.2d, v3.4s, v2.4s
+; CHECK-NEXT: uaddl2 v5.2d, v0.4s, v1.4s
+; CHECK-NEXT: uaddl v2.2d, v3.2s, v2.2s
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v1.2d, v5.2d, v4.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <16 x i8> %x to <16 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v16i8_v16i64_acc_sext(<16 x i8> %x, i64 %a) {
+; CHECK-LABEL: add_v16i8_v16i64_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll2 v1.8h, v0.16b, #0
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: sshll2 v2.4s, v1.8h, #0
+; CHECK-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-NEXT: sshll2 v3.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: saddl2 v4.2d, v3.4s, v2.4s
+; CHECK-NEXT: saddl2 v5.2d, v0.4s, v1.4s
+; CHECK-NEXT: saddl v2.2d, v3.2s, v2.2s
+; CHECK-NEXT: saddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v1.2d, v5.2d, v4.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <16 x i8> %x to <16 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v8i8_v8i64_acc_zext(<8 x i8> %x, i64 %a) {
+; CHECK-LABEL: add_v8i8_v8i64_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: uaddl2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <8 x i8> %x to <8 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v8i8_v8i64_acc_sext(<8 x i8> %x, i64 %a) {
+; CHECK-LABEL: add_v8i8_v8i64_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: sshll2 v1.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: saddl2 v2.2d, v0.4s, v1.4s
+; CHECK-NEXT: saddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v2.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <8 x i8> %x to <8 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v4i8_v4i64_acc_zext(<4 x i8> %x, i64 %a) {
+; CHECK-LABEL: add_v4i8_v4i64_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NEXT: uaddw2 v0.2d, v1.2d, v0.4s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i8> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v4i8_v4i64_acc_sext(<4 x i8> %x, i64 %a) {
+; CHECK-LABEL: add_v4i8_v4i64_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v1.2d, v0.2s, #0
+; CHECK-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NEXT: shl v1.2d, v1.2d, #56
+; CHECK-NEXT: shl v0.2d, v0.2d, #56
+; CHECK-NEXT: sshr v1.2d, v1.2d, #56
+; CHECK-NEXT: ssra v1.2d, v0.2d, #56
+; CHECK-NEXT: addp d0, v1.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i8> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v2i8_v2i64_acc_zext(<2 x i8> %x, i64 %a) {
+; CHECK-LABEL: add_v2i8_v2i64_acc_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d1, #0x0000ff000000ff
+; CHECK-NEXT: and v0.8b, v0.8b, v1.8b
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <2 x i8> %x to <2 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v2i8_v2i64_acc_sext(<2 x i8> %x, i64 %a) {
+; CHECK-LABEL: add_v2i8_v2i64_acc_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: shl v0.2d, v0.2d, #56
+; CHECK-NEXT: sshr v0.2d, v0.2d, #56
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <2 x i8> %x to <2 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i64 @add_v2i64_v2i64_acc(<2 x i64> %x, i64 %a) {
+; CHECK-LABEL: add_v2i64_v2i64_acc:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x8, d0
+; CHECK-NEXT: add x0, x8, x0
+; CHECK-NEXT: ret
+entry:
+ %z = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %x)
+ %r = add i64 %z, %a
+ ret i64 %r
+}
+
+define i32 @add_pair_v4i32_v4i32(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: add_pair_v4i32_v4i32:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %x)
+ %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %y)
+ %z = add i32 %z1, %z2
+ ret i32 %z
+}
+
+define i64 @add_pair_v4i32_v4i64_zext(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: add_pair_v4i32_v4i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-NEXT: uaddw2 v0.2d, v2.2d, v0.4s
+; CHECK-NEXT: uaddw2 v1.2d, v3.2d, v1.4s
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i32> %x to <4 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %yy = zext <4 x i32> %y to <4 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v4i32_v4i64_sext(<4 x i32> %x, <4 x i32> %y) {
+; CHECK-LABEL: add_pair_v4i32_v4i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v2.2d, v0.2s, #0
+; CHECK-NEXT: sshll v3.2d, v1.2s, #0
+; CHECK-NEXT: saddw2 v0.2d, v2.2d, v0.4s
+; CHECK-NEXT: saddw2 v1.2d, v3.2d, v1.4s
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i32> %x to <4 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %yy = sext <4 x i32> %y to <4 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v2i32_v2i64_zext(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: add_pair_v2i32_v2i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <2 x i32> %x to <2 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ %yy = zext <2 x i32> %y to <2 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v2i32_v2i64_sext(<2 x i32> %x, <2 x i32> %y) {
+; CHECK-LABEL: add_pair_v2i32_v2i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: saddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <2 x i32> %x to <2 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ %yy = sext <2 x i32> %y to <2 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i32 @add_pair_v8i16_v8i32_zext(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: add_pair_v8i16_v8i32_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-NEXT: ushll v3.4s, v1.4h, #0
+; CHECK-NEXT: uaddw2 v0.4s, v2.4s, v0.8h
+; CHECK-NEXT: uaddw2 v1.4s, v3.4s, v1.8h
+; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <8 x i16> %x to <8 x i32>
+ %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ %yy = zext <8 x i16> %y to <8 x i32>
+ %z2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %yy)
+ %z = add i32 %z1, %z2
+ ret i32 %z
+}
+
+define i32 @add_pair_v8i16_v8i32_sext(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: add_pair_v8i16_v8i32_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v2.4s, v0.4h, #0
+; CHECK-NEXT: sshll v3.4s, v1.4h, #0
+; CHECK-NEXT: saddw2 v0.4s, v2.4s, v0.8h
+; CHECK-NEXT: saddw2 v1.4s, v3.4s, v1.8h
+; CHECK-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <8 x i16> %x to <8 x i32>
+ %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ %yy = sext <8 x i16> %y to <8 x i32>
+ %z2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %yy)
+ %z = add i32 %z1, %z2
+ ret i32 %z
+}
+
+define i32 @add_pair_v4i16_v4i32_zext(<4 x i16> %x, <4 x i16> %y) {
+; CHECK-LABEL: add_pair_v4i16_v4i32_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: uaddl v0.4s, v0.4h, v1.4h
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i16> %x to <4 x i32>
+ %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ %yy = zext <4 x i16> %y to <4 x i32>
+ %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy)
+ %z = add i32 %z1, %z2
+ ret i32 %z
+}
+
+define i32 @add_pair_v4i16_v4i32_sext(<4 x i16> %x, <4 x i16> %y) {
+; CHECK-LABEL: add_pair_v4i16_v4i32_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: saddl v0.4s, v0.4h, v1.4h
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i16> %x to <4 x i32>
+ %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ %yy = sext <4 x i16> %y to <4 x i32>
+ %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy)
+ %z = add i32 %z1, %z2
+ ret i32 %z
+}
+
+define zeroext i16 @add_pair_v8i16_v8i16(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: add_pair_v8i16_v8i16:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: addv h1, v1.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: fmov w9, s1
+; CHECK-NEXT: add w8, w8, w9
+; CHECK-NEXT: and w0, w8, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %z1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %x)
+ %z2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %y)
+ %z = add i16 %z1, %z2
+ ret i16 %z
+}
+
+define i64 @add_pair_v8i16_v8i64_zext(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: add_pair_v8i16_v8i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v1.8h, #0
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: uaddl2 v4.2d, v0.4s, v2.4s
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v2.2s
+; CHECK-NEXT: uaddl2 v2.2d, v1.4s, v3.4s
+; CHECK-NEXT: uaddl v1.2d, v1.2s, v3.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v4.2d
+; CHECK-NEXT: add v1.2d, v1.2d, v2.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <8 x i16> %x to <8 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)
+ %yy = zext <8 x i16> %y to <8 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v8i16_v8i64_sext(<8 x i16> %x, <8 x i16> %y) {
+; CHECK-LABEL: add_pair_v8i16_v8i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll2 v2.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: sshll2 v3.4s, v1.8h, #0
+; CHECK-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-NEXT: saddl2 v4.2d, v0.4s, v2.4s
+; CHECK-NEXT: saddl v0.2d, v0.2s, v2.2s
+; CHECK-NEXT: saddl2 v2.2d, v1.4s, v3.4s
+; CHECK-NEXT: saddl v1.2d, v1.2s, v3.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v4.2d
+; CHECK-NEXT: add v1.2d, v1.2d, v2.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <8 x i16> %x to <8 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)
+ %yy = sext <8 x i16> %y to <8 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v4i16_v4i64_zext(<4 x i16> %x, <4 x i16> %y) {
+; CHECK-LABEL: add_pair_v4i16_v4i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-NEXT: uaddw2 v0.2d, v2.2d, v0.4s
+; CHECK-NEXT: uaddw2 v1.2d, v3.2d, v1.4s
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i16> %x to <4 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %yy = zext <4 x i16> %y to <4 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v4i16_v4i64_sext(<4 x i16> %x, <4 x i16> %y) {
+; CHECK-LABEL: add_pair_v4i16_v4i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-NEXT: sshll v2.2d, v0.2s, #0
+; CHECK-NEXT: sshll v3.2d, v1.2s, #0
+; CHECK-NEXT: saddw2 v0.2d, v2.2d, v0.4s
+; CHECK-NEXT: saddw2 v1.2d, v3.2d, v1.4s
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i16> %x to <4 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %yy = sext <4 x i16> %y to <4 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v2i16_v2i64_zext(<2 x i16> %x, <2 x i16> %y) {
+; CHECK-LABEL: add_pair_v2i16_v2i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d2, #0x00ffff0000ffff
+; CHECK-NEXT: and v0.8b, v0.8b, v2.8b
+; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <2 x i16> %x to <2 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ %yy = zext <2 x i16> %y to <2 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v2i16_v2i64_sext(<2 x i16> %x, <2 x i16> %y) {
+; CHECK-LABEL: add_pair_v2i16_v2i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: ushll v1.2d, v1.2s, #0
+; CHECK-NEXT: shl v0.2d, v0.2d, #48
+; CHECK-NEXT: shl v1.2d, v1.2d, #48
+; CHECK-NEXT: sshr v0.2d, v0.2d, #48
+; CHECK-NEXT: ssra v0.2d, v1.2d, #48
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <2 x i16> %x to <2 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ %yy = sext <2 x i16> %y to <2 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i32 @add_pair_v16i8_v16i32_zext(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-BASE-LABEL: add_pair_v16i8_v16i32_zext:
+; CHECK-BASE: // %bb.0: // %entry
+; CHECK-BASE-NEXT: ushll2 v2.8h, v0.16b, #0
+; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BASE-NEXT: ushll2 v3.8h, v1.16b, #0
+; CHECK-BASE-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-BASE-NEXT: uaddl2 v4.4s, v0.8h, v2.8h
+; CHECK-BASE-NEXT: uaddl v0.4s, v0.4h, v2.4h
+; CHECK-BASE-NEXT: uaddl2 v2.4s, v1.8h, v3.8h
+; CHECK-BASE-NEXT: uaddl v1.4s, v1.4h, v3.4h
+; CHECK-BASE-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-BASE-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-BASE-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-BASE-NEXT: addv s0, v0.4s
+; CHECK-BASE-NEXT: fmov w0, s0
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-DOT-LABEL: add_pair_v16i8_v16i32_zext:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v2.16b, #1
+; CHECK-DOT-NEXT: movi v3.2d, #0000000000000000
+; CHECK-DOT-NEXT: udot v3.4s, v1.16b, v2.16b
+; CHECK-DOT-NEXT: udot v3.4s, v0.16b, v2.16b
+; CHECK-DOT-NEXT: addv s0, v3.4s
+; CHECK-DOT-NEXT: fmov w0, s0
+; CHECK-DOT-NEXT: ret
+entry:
+ %xx = zext <16 x i8> %x to <16 x i32>
+ %z1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx)
+ %yy = zext <16 x i8> %y to <16 x i32>
+ %z2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %yy)
+ %z = add i32 %z1, %z2
+ ret i32 %z
+}
+
+define i32 @add_pair_v16i8_v16i32_sext(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-BASE-LABEL: add_pair_v16i8_v16i32_sext:
+; CHECK-BASE: // %bb.0: // %entry
+; CHECK-BASE-NEXT: sshll2 v2.8h, v0.16b, #0
+; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-BASE-NEXT: sshll2 v3.8h, v1.16b, #0
+; CHECK-BASE-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-BASE-NEXT: saddl2 v4.4s, v0.8h, v2.8h
+; CHECK-BASE-NEXT: saddl v0.4s, v0.4h, v2.4h
+; CHECK-BASE-NEXT: saddl2 v2.4s, v1.8h, v3.8h
+; CHECK-BASE-NEXT: saddl v1.4s, v1.4h, v3.4h
+; CHECK-BASE-NEXT: add v0.4s, v0.4s, v4.4s
+; CHECK-BASE-NEXT: add v1.4s, v1.4s, v2.4s
+; CHECK-BASE-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-BASE-NEXT: addv s0, v0.4s
+; CHECK-BASE-NEXT: fmov w0, s0
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-DOT-LABEL: add_pair_v16i8_v16i32_sext:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v2.16b, #1
+; CHECK-DOT-NEXT: movi v3.2d, #0000000000000000
+; CHECK-DOT-NEXT: sdot v3.4s, v1.16b, v2.16b
+; CHECK-DOT-NEXT: sdot v3.4s, v0.16b, v2.16b
+; CHECK-DOT-NEXT: addv s0, v3.4s
+; CHECK-DOT-NEXT: fmov w0, s0
+; CHECK-DOT-NEXT: ret
+entry:
+ %xx = sext <16 x i8> %x to <16 x i32>
+ %z1 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %xx)
+ %yy = sext <16 x i8> %y to <16 x i32>
+ %z2 = call i32 @llvm.vector.reduce.add.v16i32(<16 x i32> %yy)
+ %z = add i32 %z1, %z2
+ ret i32 %z
+}
+
+define i32 @add_pair_v8i8_v8i32_zext(<8 x i8> %x, <8 x i8> %y) {
+; CHECK-BASE-LABEL: add_pair_v8i8_v8i32_zext:
+; CHECK-BASE: // %bb.0: // %entry
+; CHECK-BASE-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-BASE-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-BASE-NEXT: ushll v2.4s, v0.4h, #0
+; CHECK-BASE-NEXT: ushll v3.4s, v1.4h, #0
+; CHECK-BASE-NEXT: uaddw2 v0.4s, v2.4s, v0.8h
+; CHECK-BASE-NEXT: uaddw2 v1.4s, v3.4s, v1.8h
+; CHECK-BASE-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-BASE-NEXT: addv s0, v0.4s
+; CHECK-BASE-NEXT: fmov w0, s0
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-DOT-LABEL: add_pair_v8i8_v8i32_zext:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v2.8b, #1
+; CHECK-DOT-NEXT: movi v3.2d, #0000000000000000
+; CHECK-DOT-NEXT: udot v3.2s, v1.8b, v2.8b
+; CHECK-DOT-NEXT: udot v3.2s, v0.8b, v2.8b
+; CHECK-DOT-NEXT: addp v0.2s, v3.2s, v3.2s
+; CHECK-DOT-NEXT: fmov w0, s0
+; CHECK-DOT-NEXT: ret
+entry:
+ %xx = zext <8 x i8> %x to <8 x i32>
+ %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ %yy = zext <8 x i8> %y to <8 x i32>
+ %z2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %yy)
+ %z = add i32 %z1, %z2
+ ret i32 %z
+}
+
+define i32 @add_pair_v8i8_v8i32_sext(<8 x i8> %x, <8 x i8> %y) {
+; CHECK-BASE-LABEL: add_pair_v8i8_v8i32_sext:
+; CHECK-BASE: // %bb.0: // %entry
+; CHECK-BASE-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-BASE-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-BASE-NEXT: sshll v2.4s, v0.4h, #0
+; CHECK-BASE-NEXT: sshll v3.4s, v1.4h, #0
+; CHECK-BASE-NEXT: saddw2 v0.4s, v2.4s, v0.8h
+; CHECK-BASE-NEXT: saddw2 v1.4s, v3.4s, v1.8h
+; CHECK-BASE-NEXT: add v0.4s, v0.4s, v1.4s
+; CHECK-BASE-NEXT: addv s0, v0.4s
+; CHECK-BASE-NEXT: fmov w0, s0
+; CHECK-BASE-NEXT: ret
+;
+; CHECK-DOT-LABEL: add_pair_v8i8_v8i32_sext:
+; CHECK-DOT: // %bb.0: // %entry
+; CHECK-DOT-NEXT: movi v2.8b, #1
+; CHECK-DOT-NEXT: movi v3.2d, #0000000000000000
+; CHECK-DOT-NEXT: sdot v3.2s, v1.8b, v2.8b
+; CHECK-DOT-NEXT: sdot v3.2s, v0.8b, v2.8b
+; CHECK-DOT-NEXT: addp v0.2s, v3.2s, v3.2s
+; CHECK-DOT-NEXT: fmov w0, s0
+; CHECK-DOT-NEXT: ret
+entry:
+ %xx = sext <8 x i8> %x to <8 x i32>
+ %z1 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ %yy = sext <8 x i8> %y to <8 x i32>
+ %z2 = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %yy)
+ %z = add i32 %z1, %z2
+ ret i32 %z
+}
+
+define i32 @add_pair_v4i8_v4i32_zext(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: add_pair_v4i8_v4i32_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NEXT: uaddl v0.4s, v0.4h, v1.4h
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i8> %x to <4 x i32>
+ %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ %yy = zext <4 x i8> %y to <4 x i32>
+ %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy)
+ %z = add i32 %z1, %z2
+ ret i32 %z
+}
+
+define i32 @add_pair_v4i8_v4i32_sext(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: add_pair_v4i8_v4i32_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: shl v0.4s, v0.4s, #24
+; CHECK-NEXT: shl v1.4s, v1.4s, #24
+; CHECK-NEXT: sshr v0.4s, v0.4s, #24
+; CHECK-NEXT: ssra v0.4s, v1.4s, #24
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: fmov w0, s0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i8> %x to <4 x i32>
+ %z1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ %yy = sext <4 x i8> %y to <4 x i32>
+ %z2 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %yy)
+ %z = add i32 %z1, %z2
+ ret i32 %z
+}
+
+define zeroext i16 @add_pair_v16i8_v16i16_zext(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: add_pair_v16i8_v16i16_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v2.8h, v0.8b, #0
+; CHECK-NEXT: ushll v3.8h, v1.8b, #0
+; CHECK-NEXT: uaddw2 v0.8h, v2.8h, v0.16b
+; CHECK-NEXT: uaddw2 v1.8h, v3.8h, v1.16b
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: addv h1, v1.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: fmov w9, s1
+; CHECK-NEXT: add w8, w8, w9
+; CHECK-NEXT: and w0, w8, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <16 x i8> %x to <16 x i16>
+ %z1 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)
+ %yy = zext <16 x i8> %y to <16 x i16>
+ %z2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %yy)
+ %z = add i16 %z1, %z2
+ ret i16 %z
+}
+
+define signext i16 @add_pair_v16i8_v16i16_sext(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: add_pair_v16i8_v16i16_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v2.8h, v0.8b, #0
+; CHECK-NEXT: sshll v3.8h, v1.8b, #0
+; CHECK-NEXT: saddw2 v0.8h, v2.8h, v0.16b
+; CHECK-NEXT: saddw2 v1.8h, v3.8h, v1.16b
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: addv h1, v1.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: fmov w9, s1
+; CHECK-NEXT: add w8, w8, w9
+; CHECK-NEXT: sxth w0, w8
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <16 x i8> %x to <16 x i16>
+ %z1 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)
+ %yy = sext <16 x i8> %y to <16 x i16>
+ %z2 = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %yy)
+ %z = add i16 %z1, %z2
+ ret i16 %z
+}
+
+define zeroext i16 @add_pair_v8i8_v8i16_zext(<8 x i8> %x, <8 x i8> %y) {
+; CHECK-LABEL: add_pair_v8i8_v8i16_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: addv h1, v1.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: fmov w9, s1
+; CHECK-NEXT: add w8, w8, w9
+; CHECK-NEXT: and w0, w8, #0xffff
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <8 x i8> %x to <8 x i16>
+ %z1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)
+ %yy = zext <8 x i8> %y to <8 x i16>
+ %z2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %yy)
+ %z = add i16 %z1, %z2
+ ret i16 %z
+}
+
+define signext i16 @add_pair_v8i8_v8i16_sext(<8 x i8> %x, <8 x i8> %y) {
+; CHECK-LABEL: add_pair_v8i8_v8i16_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: addv h1, v1.8h
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: fmov w9, s1
+; CHECK-NEXT: add w8, w8, w9
+; CHECK-NEXT: sxth w0, w8
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <8 x i8> %x to <8 x i16>
+ %z1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)
+ %yy = sext <8 x i8> %y to <8 x i16>
+ %z2 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %yy)
+ %z = add i16 %z1, %z2
+ ret i16 %z
+}
+
+define zeroext i8 @add_pair_v16i8_v16i8(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: add_pair_v16i8_v16i8:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: addv b0, v0.16b
+; CHECK-NEXT: addv b1, v1.16b
+; CHECK-NEXT: fmov w8, s0
+; CHECK-NEXT: fmov w9, s1
+; CHECK-NEXT: add w8, w8, w9
+; CHECK-NEXT: and w0, w8, #0xff
+; CHECK-NEXT: ret
+entry:
+ %z1 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %x)
+ %z2 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %y)
+ %z = add i8 %z1, %z2
+ ret i8 %z
+}
+
+define i64 @add_pair_v16i8_v16i64_zext(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: add_pair_v16i8_v16i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll2 v2.8h, v0.16b, #0
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ushll2 v3.8h, v1.16b, #0
+; CHECK-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NEXT: ushll2 v4.4s, v2.8h, #0
+; CHECK-NEXT: ushll v2.4s, v2.4h, #0
+; CHECK-NEXT: ushll2 v5.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v6.4s, v3.4h, #0
+; CHECK-NEXT: ushll v7.4s, v1.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v3.8h, #0
+; CHECK-NEXT: ushll2 v1.4s, v1.8h, #0
+; CHECK-NEXT: uaddl2 v16.2d, v5.4s, v4.4s
+; CHECK-NEXT: uaddl v4.2d, v5.2s, v4.2s
+; CHECK-NEXT: uaddl2 v5.2d, v0.4s, v2.4s
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v2.2s
+; CHECK-NEXT: uaddl2 v2.2d, v1.4s, v3.4s
+; CHECK-NEXT: uaddl v1.2d, v1.2s, v3.2s
+; CHECK-NEXT: uaddl2 v3.2d, v7.4s, v6.4s
+; CHECK-NEXT: uaddl v6.2d, v7.2s, v6.2s
+; CHECK-NEXT: add v5.2d, v5.2d, v16.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v4.2d
+; CHECK-NEXT: add v2.2d, v3.2d, v2.2d
+; CHECK-NEXT: add v1.2d, v6.2d, v1.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v5.2d
+; CHECK-NEXT: add v1.2d, v1.2d, v2.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <16 x i8> %x to <16 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx)
+ %yy = zext <16 x i8> %y to <16 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v16i8_v16i64_sext(<16 x i8> %x, <16 x i8> %y) {
+; CHECK-LABEL: add_pair_v16i8_v16i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll2 v2.8h, v0.16b, #0
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: sshll2 v3.8h, v1.16b, #0
+; CHECK-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-NEXT: sshll2 v4.4s, v2.8h, #0
+; CHECK-NEXT: sshll v2.4s, v2.4h, #0
+; CHECK-NEXT: sshll2 v5.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: sshll v6.4s, v3.4h, #0
+; CHECK-NEXT: sshll v7.4s, v1.4h, #0
+; CHECK-NEXT: sshll2 v3.4s, v3.8h, #0
+; CHECK-NEXT: sshll2 v1.4s, v1.8h, #0
+; CHECK-NEXT: saddl2 v16.2d, v5.4s, v4.4s
+; CHECK-NEXT: saddl v4.2d, v5.2s, v4.2s
+; CHECK-NEXT: saddl2 v5.2d, v0.4s, v2.4s
+; CHECK-NEXT: saddl v0.2d, v0.2s, v2.2s
+; CHECK-NEXT: saddl2 v2.2d, v1.4s, v3.4s
+; CHECK-NEXT: saddl v1.2d, v1.2s, v3.2s
+; CHECK-NEXT: saddl2 v3.2d, v7.4s, v6.4s
+; CHECK-NEXT: saddl v6.2d, v7.2s, v6.2s
+; CHECK-NEXT: add v5.2d, v5.2d, v16.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v4.2d
+; CHECK-NEXT: add v2.2d, v3.2d, v2.2d
+; CHECK-NEXT: add v1.2d, v6.2d, v1.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v5.2d
+; CHECK-NEXT: add v1.2d, v1.2d, v2.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <16 x i8> %x to <16 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %xx)
+ %yy = sext <16 x i8> %y to <16 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v16i64(<16 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v8i8_v8i64_zext(<8 x i8> %x, <8 x i8> %y) {
+; CHECK-LABEL: add_pair_v8i8_v8i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.8h, v0.8b, #0
+; CHECK-NEXT: ushll v1.8h, v1.8b, #0
+; CHECK-NEXT: ushll2 v2.4s, v0.8h, #0
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll2 v3.4s, v1.8h, #0
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: uaddl2 v4.2d, v0.4s, v2.4s
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v2.2s
+; CHECK-NEXT: uaddl2 v2.2d, v1.4s, v3.4s
+; CHECK-NEXT: uaddl v1.2d, v1.2s, v3.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v4.2d
+; CHECK-NEXT: add v1.2d, v1.2d, v2.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <8 x i8> %x to <8 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)
+ %yy = zext <8 x i8> %y to <8 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v8i8_v8i64_sext(<8 x i8> %x, <8 x i8> %y) {
+; CHECK-LABEL: add_pair_v8i8_v8i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: sshll v0.8h, v0.8b, #0
+; CHECK-NEXT: sshll v1.8h, v1.8b, #0
+; CHECK-NEXT: sshll2 v2.4s, v0.8h, #0
+; CHECK-NEXT: sshll v0.4s, v0.4h, #0
+; CHECK-NEXT: sshll2 v3.4s, v1.8h, #0
+; CHECK-NEXT: sshll v1.4s, v1.4h, #0
+; CHECK-NEXT: saddl2 v4.2d, v0.4s, v2.4s
+; CHECK-NEXT: saddl v0.2d, v0.2s, v2.2s
+; CHECK-NEXT: saddl2 v2.2d, v1.4s, v3.4s
+; CHECK-NEXT: saddl v1.2d, v1.2s, v3.2s
+; CHECK-NEXT: add v0.2d, v0.2d, v4.2d
+; CHECK-NEXT: add v1.2d, v1.2d, v2.2d
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <8 x i8> %x to <8 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %xx)
+ %yy = sext <8 x i8> %y to <8 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v8i64(<8 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v4i8_v4i64_zext(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: add_pair_v4i8_v4i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: bic v0.4h, #255, lsl #8
+; CHECK-NEXT: bic v1.4h, #255, lsl #8
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-NEXT: uaddw2 v0.2d, v2.2d, v0.4s
+; CHECK-NEXT: uaddw2 v1.2d, v3.2d, v1.4s
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <4 x i8> %x to <4 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %yy = zext <4 x i8> %y to <4 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v4i8_v4i64_sext(<4 x i8> %x, <4 x i8> %y) {
+; CHECK-LABEL: add_pair_v4i8_v4i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.4s, v0.4h, #0
+; CHECK-NEXT: ushll v1.4s, v1.4h, #0
+; CHECK-NEXT: ushll v2.2d, v0.2s, #0
+; CHECK-NEXT: ushll v3.2d, v1.2s, #0
+; CHECK-NEXT: ushll2 v0.2d, v0.4s, #0
+; CHECK-NEXT: ushll2 v1.2d, v1.4s, #0
+; CHECK-NEXT: shl v2.2d, v2.2d, #56
+; CHECK-NEXT: shl v3.2d, v3.2d, #56
+; CHECK-NEXT: shl v0.2d, v0.2d, #56
+; CHECK-NEXT: shl v1.2d, v1.2d, #56
+; CHECK-NEXT: sshr v2.2d, v2.2d, #56
+; CHECK-NEXT: sshr v3.2d, v3.2d, #56
+; CHECK-NEXT: ssra v2.2d, v0.2d, #56
+; CHECK-NEXT: ssra v3.2d, v1.2d, #56
+; CHECK-NEXT: add v0.2d, v2.2d, v3.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <4 x i8> %x to <4 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %yy = sext <4 x i8> %y to <4 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v2i8_v2i64_zext(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: add_pair_v2i8_v2i64_zext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d2, #0x0000ff000000ff
+; CHECK-NEXT: and v0.8b, v0.8b, v2.8b
+; CHECK-NEXT: and v1.8b, v1.8b, v2.8b
+; CHECK-NEXT: uaddl v0.2d, v0.2s, v1.2s
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = zext <2 x i8> %x to <2 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ %yy = zext <2 x i8> %y to <2 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v2i8_v2i64_sext(<2 x i8> %x, <2 x i8> %y) {
+; CHECK-LABEL: add_pair_v2i8_v2i64_sext:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ushll v0.2d, v0.2s, #0
+; CHECK-NEXT: ushll v1.2d, v1.2s, #0
+; CHECK-NEXT: shl v0.2d, v0.2d, #56
+; CHECK-NEXT: shl v1.2d, v1.2d, #56
+; CHECK-NEXT: sshr v0.2d, v0.2d, #56
+; CHECK-NEXT: ssra v0.2d, v1.2d, #56
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %xx = sext <2 x i8> %x to <2 x i64>
+ %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %xx)
+ %yy = sext <2 x i8> %y to <2 x i64>
+ %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %yy)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+define i64 @add_pair_v2i64_v2i64(<2 x i64> %x, <2 x i64> %y) {
+; CHECK-LABEL: add_pair_v2i64_v2i64:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: add v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: addp d0, v0.2d
+; CHECK-NEXT: fmov x0, d0
+; CHECK-NEXT: ret
+entry:
+ %z1 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %x)
+ %z2 = call i64 @llvm.vector.reduce.add.v2i64(<2 x i64> %y)
+ %z = add i64 %z1, %z2
+ ret i64 %z
+}
+
+declare i16 @llvm.vector.reduce.add.v16i16(<16 x i16>)
+declare i16 @llvm.vector.reduce.add.v8i16(<8 x i16>)
+declare i32 @llvm.vector.reduce.add.v16i32(<16 x i32>)
+declare i32 @llvm.vector.reduce.add.v4i32(<4 x i32>)
+declare i32 @llvm.vector.reduce.add.v8i32(<8 x i32>)
+declare i64 @llvm.vector.reduce.add.v16i64(<16 x i64>)
+declare i64 @llvm.vector.reduce.add.v2i64(<2 x i64>)
+declare i64 @llvm.vector.reduce.add.v4i64(<4 x i64>)
+declare i64 @llvm.vector.reduce.add.v8i64(<8 x i64>)
+declare i8 @llvm.vector.reduce.add.v16i8(<16 x i8>)
More information about the llvm-commits
mailing list