[llvm] [AArch64][SelectionDAG] Optimise ADDLV reductions inserted into zero vector (PR #215814)
Kieran B via llvm-commits
llvm-commits at lists.llvm.org
Wed Aug 12 07:48:13 PDT 2026
https://github.com/kieroxide created https://github.com/llvm/llvm-project/pull/215814
{U/S}ADDLV instruction inserts result into lane 0 and clear unused lanes to zero, so we can just return the result as a vector without extra insert into a zeroed vector.
Add patterns that recognize the redundant zero vector insert and remove it.
>From fc0063ad690e57dc84bb1df33413ebefa4e4da52 Mon Sep 17 00:00:00 2001
From: Kieran Bailey <kieran.bailey at arm.com>
Date: Wed, 12 Aug 2026 10:26:22 +0000
Subject: [PATCH] [AArch64][SelectionDAG] Optimise ADDLV reductions inserted
into zero vector
{U/S}ADDLV insert result into lane 0 and clear unused lanes, so we can just return the result as a vector without extra zero vector insert instructions.
Add patterns that recognise the redundant zero vector insert and remove it.
---
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 67 ++-
.../CodeGen/AArch64/vecreduce-add-vector.ll | 423 ++++++++++++++++++
2 files changed, 478 insertions(+), 12 deletions(-)
create mode 100644 llvm/test/CodeGen/AArch64/vecreduce-add-vector.ll
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 53c7af890b6e5..5b8f596e39483 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -8386,18 +8386,37 @@ defm FMAXV : SIMDFPAcrossLanes<0b01111, 0, "fmaxv", AArch64fmaxv>;
defm FMINNMV : SIMDFPAcrossLanes<0b01100, 1, "fminnmv", AArch64fminnmv>;
defm FMINV : SIMDFPAcrossLanes<0b01111, 1, "fminv", AArch64fminv>;
-def : Pat<(i32 (vector_extract (v8i16 (AArch64uaddv (v8i16 (sext (v8i8 V64:$op))))), (i64 0))),
- (EXTRACT_SUBREG (v8i16 (SUBREG_TO_REG (SADDLVv8i8v V64:$op), hsub)), ssub)>;
-def : Pat<(i32 (vector_extract (v8i16 (AArch64uaddv (v8i16 (zext (v8i8 V64:$op))))), (i64 0))),
- (EXTRACT_SUBREG (v8i16 (SUBREG_TO_REG (UADDLVv8i8v V64:$op), hsub)), ssub)>;
-def : Pat<(v8i16 (AArch64uaddv (v8i16 (sext (v8i8 V64:$op))))),
- (v8i16 (SUBREG_TO_REG (SADDLVv8i8v V64:$op), hsub))>;
-def : Pat<(v8i16 (AArch64uaddv (v8i16 (zext (v8i8 V64:$op))))),
- (v8i16 (SUBREG_TO_REG (UADDLVv8i8v V64:$op), hsub))>;
-def : Pat<(v4i32 (AArch64uaddv (v4i32 (sext (v4i16 V64:$op))))),
- (v4i32 (SUBREG_TO_REG (SADDLVv4i16v V64:$op), ssub))>;
-def : Pat<(v4i32 (AArch64uaddv (v4i32 (zext (v4i16 V64:$op))))),
- (v4i32 (SUBREG_TO_REG (UADDLVv4i16v V64:$op), ssub))>;
+multiclass SIMDAcrossLaneLongReductionExt<string Opc, SDPatternOperator ext> {
+ def : Pat<(i32 (vector_extract (v8i16 (AArch64uaddv (v8i16 (ext (v8i8 V64:$op))))), (i64 0))),
+ (EXTRACT_SUBREG (v8i16 (SUBREG_TO_REG (!cast<Instruction>(Opc#"v8i8v") V64:$op), hsub)), ssub)>;
+
+ def : Pat<(v8i16 (AArch64uaddv (v8i16 (ext (v8i8 V64:$op))))),
+ (v8i16 (SUBREG_TO_REG (!cast<Instruction>(Opc#"v8i8v") V64:$op), hsub))>;
+ def : Pat<(v4i32 (AArch64uaddv (v4i32 (ext (v4i16 V64:$op))))),
+ (v4i32 (SUBREG_TO_REG (!cast<Instruction>(Opc#"v4i16v") V64:$op), ssub))>;
+
+ // ADDLV reduction unused lanes are set to zero. Inserting into a zero vector is redundant
+ def : Pat<(v4i16 (vector_insert (v4i16 immAllZerosV),
+ (i32 (vector_extract (v8i16 (AArch64uaddv (v8i16 (ext (v8i8 V64:$Rn))))),
+ (i64 0))), (i64 0))),
+ (v4i16 (SUBREG_TO_REG (!cast<Instruction>(Opc#"v8i8v") V64:$Rn), hsub))>;
+ def : Pat<(v8i16 (vector_insert (v8i16 immAllZerosV),
+ (i32 (vector_extract (v8i16 (AArch64uaddv (v8i16 (ext (v8i8 V64:$Rn))))),
+ (i64 0))), (i64 0))),
+ (v8i16 (SUBREG_TO_REG (!cast<Instruction>(Opc#"v8i8v") V64:$Rn), hsub))>;
+ def : Pat<(v2i32 (vector_insert (v2i32 immAllZerosV),
+ (i32 (vector_extract (v4i32 (AArch64uaddv (v4i32 (ext (v4i16 V64:$Rn))))),
+ (i64 0))), (i64 0))),
+ (v2i32 (SUBREG_TO_REG (!cast<Instruction>(Opc#"v4i16v") V64:$Rn), ssub))>;
+ def : Pat<(v4i32 (vector_insert (v4i32 immAllZerosV),
+ (i32 (vector_extract (v4i32 (AArch64uaddv (v4i32 (ext (v4i16 V64:$Rn))))),
+ (i64 0))), (i64 0))),
+ (v4i32 (SUBREG_TO_REG (!cast<Instruction>(Opc#"v4i16v") V64:$Rn), ssub))>;
+}
+
+defm : SIMDAcrossLaneLongReductionExt<"UADDLV", zext>;
+defm : SIMDAcrossLaneLongReductionExt<"UADDLV", anyext>;
+defm : SIMDAcrossLaneLongReductionExt<"SADDLV", sext>;
multiclass SIMDAcrossLaneLongPairIntrinsic<string Opc, SDPatternOperator addlp> {
// Patterns for addv(addlp(x)) ==> addlv
@@ -8412,6 +8431,30 @@ multiclass SIMDAcrossLaneLongPairIntrinsic<string Opc, SDPatternOperator addlp>
def : Pat<(v4i32 (AArch64uaddv (v4i32 (addlp (v8i16 V128:$op))))),
(INSERT_SUBREG (v4i32 (IMPLICIT_DEF)), (!cast<Instruction>(Opc#"v8i16v") V128:$op), ssub)>;
+ // ADDLV reduction's unused lanes are set to zero. Inserting into a zero vector is redundant
+ def : Pat<(v4i16 (vector_insert (v4i16 immAllZerosV),
+ (i32 (vector_extract (v8i16 (AArch64uaddv (v8i16 (addlp (v16i8 V128:$Rn))))),
+ (i64 0))), (i64 0))),
+ (v4i16 (SUBREG_TO_REG (!cast<Instruction>(Opc#"v16i8v") V128:$Rn), hsub))>;
+ def : Pat<(v8i16 (vector_insert (v8i16 immAllZerosV),
+ (i32 (vector_extract (v8i16 (AArch64uaddv (v8i16 (addlp (v16i8 V128:$Rn))))),
+ (i64 0))), (i64 0))),
+ (v8i16 (SUBREG_TO_REG (!cast<Instruction>(Opc#"v16i8v") V128:$Rn), hsub))>;
+ def : Pat<(v2i32 (vector_insert (v2i32 immAllZerosV),
+ (i32 (vector_extract (v4i32 (AArch64uaddv (v4i32 (addlp (v8i16 V128:$Rn))))),
+ (i64 0))), (i64 0))),
+ (v2i32 (SUBREG_TO_REG (!cast<Instruction>(Opc#"v8i16v") V128:$Rn), ssub))>;
+ def : Pat<(v4i32 (vector_insert (v4i32 immAllZerosV),
+ (i32 (vector_extract (v4i32 (AArch64uaddv (v4i32 (addlp (v8i16 V128:$Rn))))),
+ (i64 0))), (i64 0))),
+ (v4i32 (SUBREG_TO_REG (!cast<Instruction>(Opc#"v8i16v") V128:$Rn), ssub))>;
+ def : Pat<(v1i64 (extract_subvector (v2i64 (AArch64uaddv (v2i64 (addlp (v4i32 V128:$Rn))))), (i64 0))),
+ (v1i64 (!cast<Instruction>(Opc#"v4i32v") V128:$Rn))>;
+ def : Pat<(v2i64 (concat_vectors
+ (v1i64 (extract_subvector (v2i64 (AArch64uaddv (v2i64 (addlp (v4i32 V128:$Rn))))), (i64 0))),
+ (v1i64 immAllZerosV))),
+ (v2i64 (SUBREG_TO_REG (!cast<Instruction>(Opc#"v4i32v") V128:$Rn), dsub))>;
+
// Patterns for addp(addlp(x)) ==> addlv
def : Pat<(v2i32 (AArch64uaddv (v2i32 (addlp (v4i16 V64:$op))))),
(INSERT_SUBREG (v2i32 (IMPLICIT_DEF)), (!cast<Instruction>(Opc#"v4i16v") V64:$op), ssub)>;
diff --git a/llvm/test/CodeGen/AArch64/vecreduce-add-vector.ll b/llvm/test/CodeGen/AArch64/vecreduce-add-vector.ll
new file mode 100644
index 0000000000000..9546c4e077de8
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/vecreduce-add-vector.ll
@@ -0,0 +1,423 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64-none-linux-gnu %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-none-linux-gnu -global-isel %s | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+define <4 x i16> @reduce_add_v8i8_zext_ret_v4i16(<8 x i8> %x) {
+; CHECK-SD-LABEL: reduce_add_v8i8_zext_ret_v4i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: uaddlv h0, v0.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v8i8_zext_ret_v4i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uaddlv h0, v0.8b
+; CHECK-GI-NEXT: mov v0.h[1], wzr
+; CHECK-GI-NEXT: mov v0.h[2], wzr
+; CHECK-GI-NEXT: mov v0.h[3], wzr
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = zext <8 x i8> %x to <8 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)
+ %t2 = insertelement <4 x i16> zeroinitializer, i16 %z, i64 0
+ ret <4 x i16> %t2
+}
+
+define <8 x i16> @reduce_add_v8i8_zext_ret_v8i16(<8 x i8> %x) {
+; CHECK-SD-LABEL: reduce_add_v8i8_zext_ret_v8i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: uaddlv h0, v0.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v8i8_zext_ret_v8i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uaddlv h0, v0.8b
+; CHECK-GI-NEXT: mov v0.h[1], wzr
+; CHECK-GI-NEXT: mov v0.h[2], wzr
+; CHECK-GI-NEXT: mov v0.h[3], wzr
+; CHECK-GI-NEXT: mov v0.h[4], wzr
+; CHECK-GI-NEXT: mov v0.h[5], wzr
+; CHECK-GI-NEXT: mov v0.h[6], wzr
+; CHECK-GI-NEXT: mov v0.h[7], wzr
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = zext <8 x i8> %x to <8 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)
+ %t2 = insertelement <8 x i16> zeroinitializer, i16 %z, i64 0
+ ret <8 x i16> %t2
+}
+
+define <4 x i16> @reduce_add_v16i8_zext_ret_v4i16(<16 x i8> %x) {
+; CHECK-SD-LABEL: reduce_add_v16i8_zext_ret_v4i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: uaddlv h0, v0.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v16i8_zext_ret_v4i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uaddlv h0, v0.16b
+; CHECK-GI-NEXT: mov v0.h[1], wzr
+; CHECK-GI-NEXT: mov v0.h[2], wzr
+; CHECK-GI-NEXT: mov v0.h[3], wzr
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = zext <16 x i8> %x to <16 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)
+ %t2 = insertelement <4 x i16> zeroinitializer, i16 %z, i64 0
+ ret <4 x i16> %t2
+}
+
+define <8 x i16> @reduce_add_v16i8_zext_ret_v8i16(<16 x i8> %x) {
+; CHECK-SD-LABEL: reduce_add_v16i8_zext_ret_v8i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: uaddlv h0, v0.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v16i8_zext_ret_v8i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uaddlv h0, v0.16b
+; CHECK-GI-NEXT: mov v0.h[1], wzr
+; CHECK-GI-NEXT: mov v0.h[2], wzr
+; CHECK-GI-NEXT: mov v0.h[3], wzr
+; CHECK-GI-NEXT: mov v0.h[4], wzr
+; CHECK-GI-NEXT: mov v0.h[5], wzr
+; CHECK-GI-NEXT: mov v0.h[6], wzr
+; CHECK-GI-NEXT: mov v0.h[7], wzr
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = zext <16 x i8> %x to <16 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)
+ %t2 = insertelement <8 x i16> zeroinitializer, i16 %z, i64 0
+ ret <8 x i16> %t2
+}
+
+define <2 x i32> @reduce_add_v4i16_zext_ret_v2i32(<4 x i16> %x) {
+; CHECK-SD-LABEL: reduce_add_v4i16_zext_ret_v2i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: uaddlv s0, v0.4h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v4i16_zext_ret_v2i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uaddlv s0, v0.4h
+; CHECK-GI-NEXT: mov v0.s[0], v0.s[0]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = zext <4 x i16> %x to <4 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ %t2 = insertelement <2 x i32> zeroinitializer, i32 %z, i64 0
+ ret <2 x i32> %t2
+}
+
+define <4 x i32> @reduce_add_v4i16_zext_ret_v4i32(<4 x i16> %x) {
+; CHECK-SD-LABEL: reduce_add_v4i16_zext_ret_v4i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: uaddlv s0, v0.4h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v4i16_zext_ret_v4i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uaddlv s0, v0.4h
+; CHECK-GI-NEXT: mov s0, v0.s[0]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: mov v0.s[2], wzr
+; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = zext <4 x i16> %x to <4 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ %t2 = insertelement <4 x i32> zeroinitializer, i32 %z, i64 0
+ ret <4 x i32> %t2
+}
+
+define <2 x i32> @reduce_add_v8i16_zext_ret_v2i32(<8 x i16> %x) {
+; CHECK-SD-LABEL: reduce_add_v8i16_zext_ret_v2i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: uaddlv s0, v0.8h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v8i16_zext_ret_v2i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uaddlv s0, v0.8h
+; CHECK-GI-NEXT: mov v0.s[0], v0.s[0]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = zext <8 x i16> %x to <8 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ %t2 = insertelement <2 x i32> zeroinitializer, i32 %z, i64 0
+ ret <2 x i32> %t2
+}
+
+define <4 x i32> @reduce_add_v8i16_zext_ret_v4i32(<8 x i16> %x) {
+; CHECK-SD-LABEL: reduce_add_v8i16_zext_ret_v4i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: uaddlv s0, v0.8h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v8i16_zext_ret_v4i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uaddlv s0, v0.8h
+; CHECK-GI-NEXT: mov s0, v0.s[0]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: mov v0.s[2], wzr
+; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = zext <8 x i16> %x to <8 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ %t2 = insertelement <4 x i32> zeroinitializer, i32 %z, i64 0
+ ret <4 x i32> %t2
+}
+
+define <1 x i64> @reduce_add_v4i32_zext_ret_v1i64(<4 x i32> %x) {
+; CHECK-SD-LABEL: reduce_add_v4i32_zext_ret_v1i64:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: uaddlv d0, v0.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v4i32_zext_ret_v1i64:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uaddlv d0, v0.4s
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = zext <4 x i32> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %t2 = insertelement <1 x i64> zeroinitializer, i64 %z, i64 0
+ ret <1 x i64> %t2
+}
+
+define <2 x i64> @reduce_add_v4i32_zext_ret_v2i64(<4 x i32> %x) {
+; CHECK-SD-LABEL: reduce_add_v4i32_zext_ret_v2i64:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: uaddlv d0, v0.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v4i32_zext_ret_v2i64:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: uaddlv d0, v0.4s
+; CHECK-GI-NEXT: mov d0, v0.d[0]
+; CHECK-GI-NEXT: mov v0.d[1], xzr
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = zext <4 x i32> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %t2 = insertelement <2 x i64> zeroinitializer, i64 %z, i64 0
+ ret <2 x i64> %t2
+}
+
+define <4 x i16> @reduce_add_v8i8_sext_ret_v4i16(<8 x i8> %x) {
+; CHECK-SD-LABEL: reduce_add_v8i8_sext_ret_v4i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: saddlv h0, v0.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v8i8_sext_ret_v4i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: saddlv h0, v0.8b
+; CHECK-GI-NEXT: mov v0.h[1], wzr
+; CHECK-GI-NEXT: mov v0.h[2], wzr
+; CHECK-GI-NEXT: mov v0.h[3], wzr
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = sext <8 x i8> %x to <8 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)
+ %t2 = insertelement <4 x i16> zeroinitializer, i16 %z, i64 0
+ ret <4 x i16> %t2
+}
+
+define <8 x i16> @reduce_add_v8i8_sext_ret_v8i16(<8 x i8> %x) {
+; CHECK-SD-LABEL: reduce_add_v8i8_sext_ret_v8i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: saddlv h0, v0.8b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v8i8_sext_ret_v8i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: saddlv h0, v0.8b
+; CHECK-GI-NEXT: mov v0.h[1], wzr
+; CHECK-GI-NEXT: mov v0.h[2], wzr
+; CHECK-GI-NEXT: mov v0.h[3], wzr
+; CHECK-GI-NEXT: mov v0.h[4], wzr
+; CHECK-GI-NEXT: mov v0.h[5], wzr
+; CHECK-GI-NEXT: mov v0.h[6], wzr
+; CHECK-GI-NEXT: mov v0.h[7], wzr
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = sext <8 x i8> %x to <8 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %xx)
+ %t2 = insertelement <8 x i16> zeroinitializer, i16 %z, i64 0
+ ret <8 x i16> %t2
+}
+
+define <4 x i16> @reduce_add_v16i8_sext_ret_v4i16(<16 x i8> %x) {
+; CHECK-SD-LABEL: reduce_add_v16i8_sext_ret_v4i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: saddlv h0, v0.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v16i8_sext_ret_v4i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: saddlv h0, v0.16b
+; CHECK-GI-NEXT: mov v0.h[1], wzr
+; CHECK-GI-NEXT: mov v0.h[2], wzr
+; CHECK-GI-NEXT: mov v0.h[3], wzr
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = sext <16 x i8> %x to <16 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)
+ %t2 = insertelement <4 x i16> zeroinitializer, i16 %z, i64 0
+ ret <4 x i16> %t2
+}
+
+define <8 x i16> @reduce_add_v16i8_sext_ret_v8i16(<16 x i8> %x) {
+; CHECK-SD-LABEL: reduce_add_v16i8_sext_ret_v8i16:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: saddlv h0, v0.16b
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v16i8_sext_ret_v8i16:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: saddlv h0, v0.16b
+; CHECK-GI-NEXT: mov v0.h[1], wzr
+; CHECK-GI-NEXT: mov v0.h[2], wzr
+; CHECK-GI-NEXT: mov v0.h[3], wzr
+; CHECK-GI-NEXT: mov v0.h[4], wzr
+; CHECK-GI-NEXT: mov v0.h[5], wzr
+; CHECK-GI-NEXT: mov v0.h[6], wzr
+; CHECK-GI-NEXT: mov v0.h[7], wzr
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = sext <16 x i8> %x to <16 x i16>
+ %z = call i16 @llvm.vector.reduce.add.v16i16(<16 x i16> %xx)
+ %t2 = insertelement <8 x i16> zeroinitializer, i16 %z, i64 0
+ ret <8 x i16> %t2
+}
+
+define <2 x i32> @reduce_add_v4i16_sext_ret_v2i32(<4 x i16> %x) {
+; CHECK-SD-LABEL: reduce_add_v4i16_sext_ret_v2i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: saddlv s0, v0.4h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v4i16_sext_ret_v2i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: saddlv s0, v0.4h
+; CHECK-GI-NEXT: mov v0.s[0], v0.s[0]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = sext <4 x i16> %x to <4 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ %t2 = insertelement <2 x i32> zeroinitializer, i32 %z, i64 0
+ ret <2 x i32> %t2
+}
+
+define <4 x i32> @reduce_add_v4i16_sext_ret_v4i32(<4 x i16> %x) {
+; CHECK-SD-LABEL: reduce_add_v4i16_sext_ret_v4i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: saddlv s0, v0.4h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v4i16_sext_ret_v4i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: saddlv s0, v0.4h
+; CHECK-GI-NEXT: mov s0, v0.s[0]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: mov v0.s[2], wzr
+; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = sext <4 x i16> %x to <4 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %xx)
+ %t2 = insertelement <4 x i32> zeroinitializer, i32 %z, i64 0
+ ret <4 x i32> %t2
+}
+
+define <2 x i32> @reduce_add_v8i16_sext_ret_v2i32(<8 x i16> %x) {
+; CHECK-SD-LABEL: reduce_add_v8i16_sext_ret_v2i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: saddlv s0, v0.8h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v8i16_sext_ret_v2i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: saddlv s0, v0.8h
+; CHECK-GI-NEXT: mov v0.s[0], v0.s[0]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = sext <8 x i16> %x to <8 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ %t2 = insertelement <2 x i32> zeroinitializer, i32 %z, i64 0
+ ret <2 x i32> %t2
+}
+
+define <4 x i32> @reduce_add_v8i16_sext_ret_v4i32(<8 x i16> %x) {
+; CHECK-SD-LABEL: reduce_add_v8i16_sext_ret_v4i32:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: saddlv s0, v0.8h
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v8i16_sext_ret_v4i32:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: saddlv s0, v0.8h
+; CHECK-GI-NEXT: mov s0, v0.s[0]
+; CHECK-GI-NEXT: mov v0.s[1], wzr
+; CHECK-GI-NEXT: mov v0.s[2], wzr
+; CHECK-GI-NEXT: mov v0.s[3], wzr
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = sext <8 x i16> %x to <8 x i32>
+ %z = call i32 @llvm.vector.reduce.add.v8i32(<8 x i32> %xx)
+ %t2 = insertelement <4 x i32> zeroinitializer, i32 %z, i64 0
+ ret <4 x i32> %t2
+}
+
+define <1 x i64> @reduce_add_v4i32_sext_ret_v1i64(<4 x i32> %x) {
+; CHECK-SD-LABEL: reduce_add_v4i32_sext_ret_v1i64:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: saddlv d0, v0.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v4i32_sext_ret_v1i64:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: saddlv d0, v0.4s
+; CHECK-GI-NEXT: // kill: def $d0 killed $d0 killed $q0
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = sext <4 x i32> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %t2 = insertelement <1 x i64> zeroinitializer, i64 %z, i64 0
+ ret <1 x i64> %t2
+}
+
+define <2 x i64> @reduce_add_v4i32_sext_ret_v2i64(<4 x i32> %x) {
+; CHECK-SD-LABEL: reduce_add_v4i32_sext_ret_v2i64:
+; CHECK-SD: // %bb.0: // %entry
+; CHECK-SD-NEXT: saddlv d0, v0.4s
+; CHECK-SD-NEXT: ret
+;
+; CHECK-GI-LABEL: reduce_add_v4i32_sext_ret_v2i64:
+; CHECK-GI: // %bb.0: // %entry
+; CHECK-GI-NEXT: saddlv d0, v0.4s
+; CHECK-GI-NEXT: mov d0, v0.d[0]
+; CHECK-GI-NEXT: mov v0.d[1], xzr
+; CHECK-GI-NEXT: ret
+entry:
+ %xx = sext <4 x i32> %x to <4 x i64>
+ %z = call i64 @llvm.vector.reduce.add.v4i64(<4 x i64> %xx)
+ %t2 = insertelement <2 x i64> zeroinitializer, i64 %z, i64 0
+ ret <2 x i64> %t2
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK: {{.*}}
More information about the llvm-commits
mailing list