[llvm] [AArch64][NEON] Fold insert(zero, extract(X, 0), 0) -> X when X is known to zero lanes 1-N (PR #213940)
via llvm-commits
llvm-commits at lists.llvm.org
Fri Aug 7 09:20:22 PDT 2026
https://github.com/CarolineConcatto updated https://github.com/llvm/llvm-project/pull/213940
>From 109ce584f8b591d6e1a228205f928cf751dd085c Mon Sep 17 00:00:00 2001
From: CarolineConcatto <caroline.concatto at arm.com>
Date: Thu, 30 Jul 2026 10:41:37 +0000
Subject: [PATCH] [AArch64][NEON] Fold insert(zero, extract(X, 0), 0) -> X when
X is known to zero lanes 1-N
Add patterns for integer NEON across-lane reductions whose scalar result
is inserted into lane zero of a zero vector.
This is valid because these instructions place their result in lane zero and
clear the remaining lanes.
---
llvm/lib/Target/AArch64/AArch64InstrInfo.td | 34 +-
.../AArch64/neon-implicit-zero-filling.ll | 327 ++++++++++++++++++
2 files changed, 360 insertions(+), 1 deletion(-)
create mode 100644 llvm/test/CodeGen/AArch64/neon-implicit-zero-filling.ll
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 30b621bab41aa..05a481bebd8fc 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -7777,7 +7777,6 @@ def : Pat<(f32 (vecreduce_fadd (v2f32 V64:$Rn))),
(FADDPv2i32p V64:$Rn)>;
def : Pat<(f64 (vecreduce_fadd (v2f64 V128:$Rn))),
(FADDPv2i64p V128:$Rn)>;
-
def : Pat<(v2i64 (AArch64saddv V128:$Rn)),
(INSERT_SUBREG (v2i64 (IMPLICIT_DEF)), (ADDPv2i64p V128:$Rn), dsub)>;
def : Pat<(v2i64 (AArch64uaddv V128:$Rn)),
@@ -8468,6 +8467,39 @@ defm : SIMDAcrossLaneLongReductionIntrinsic<"SADDLV", AArch64saddlv>;
// In effect, opNode is the same as (scalar_to_vector (IntNode)).
multiclass SIMDAcrossLanesIntrinsic<string baseOpc,
SDPatternOperator opNode> {
+// Reductions have the unused lanes set to zero. Inserting the reduction's
+// result into lane zero of a zero vector is therefore redundant.
+def : Pat<(v16i8 (vector_insert immAllZerosV,
+ (i32 (vector_extract (v16i8 (opNode V128:$Rn)), (i64 0))),
+ (i64 0))),
+ (v16i8 (INSERT_SUBREG (v16i8 (IMPLICIT_DEF)),
+ (!cast<Instruction>(!strconcat(baseOpc, "v16i8v")) V128:$Rn),
+ bsub))>;
+def : Pat<(v8i16 (vector_insert immAllZerosV,
+ (i32 (vector_extract (v8i16 (opNode V128:$Rn)), (i64 0))),
+ (i64 0))),
+ (v8i16 (INSERT_SUBREG (v8i16 (IMPLICIT_DEF)),
+ (!cast<Instruction>(!strconcat(baseOpc, "v8i16v")) V128:$Rn),
+ hsub))>;
+def : Pat<(v4i32 (vector_insert immAllZerosV,
+ (i32 (vector_extract (v4i32 (opNode V128:$Rn)), (i64 0))),
+ (i64 0))),
+ (v4i32 (INSERT_SUBREG (v4i32 (IMPLICIT_DEF)),
+ (!cast<Instruction>(!strconcat(baseOpc, "v4i32v")) V128:$Rn),
+ ssub))>;
+
+def : Pat<(v8i8 (vector_insert immAllZerosV,
+ (i32 (vector_extract (v16i8 (insert_subvector undef,
+ (v8i8 (opNode V64:$Rn)), (i64 0))), (i64 0))), (i64 0))),
+ (v8i8 (INSERT_SUBREG (v8i8 (IMPLICIT_DEF)),
+ (!cast<Instruction>(!strconcat(baseOpc, "v8i8v")) V64:$Rn), bsub))>;
+
+def : Pat<(v4i16 (vector_insert immAllZerosV,
+ (i32 (vector_extract (v8i16 (insert_subvector undef,
+ (v4i16 (opNode V64:$Rn)), (i64 0))), (i64 0))), (i64 0))),
+ (v4i16 (INSERT_SUBREG (v4i16 (IMPLICIT_DEF)),
+ (!cast<Instruction>(!strconcat(baseOpc, "v4i16v")) V64:$Rn), hsub))>;
+
// If a lane instruction caught the vector_extract around opNode, we can
// directly match the latter to the instruction.
def : Pat<(v8i8 (opNode V64:$Rn)),
diff --git a/llvm/test/CodeGen/AArch64/neon-implicit-zero-filling.ll b/llvm/test/CodeGen/AArch64/neon-implicit-zero-filling.ll
new file mode 100644
index 0000000000000..04b654e6a75f5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-implicit-zero-filling.ll
@@ -0,0 +1,327 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s | FileCheck %s
+
+target triple = "aarch64-unknown-linux-gnu"
+
+;; The following tests are the LLVM IR of this function. The example uses
+;; vector_reduce_add, but this applies to the other vector reductions as well.
+;vec_sz_out uaddvq_sz_zero_fill(vec_sz_in a) {
+; return (vec_sz_out){vaddvq_u8(a)};
+;}
+
+
+define <8 x i8> @reduce_add_v8i8_zero_fill(<8 x i8> %a) {
+; CHECK-LABEL: reduce_add_v8i8_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: addv b0, v0.8b
+; CHECK-NEXT: ret
+ %t1 = call i8 @llvm.vector.reduce.add.v8i8(<8 x i8> %a)
+ %t2 = insertelement <8 x i8> zeroinitializer, i8 %t1, i64 0
+ ret <8 x i8> %t2
+}
+
+define <8 x i8> @reduce_smax_v8i8_zero_fill(<8 x i8> %a) {
+; CHECK-LABEL: reduce_smax_v8i8_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smaxv b0, v0.8b
+; CHECK-NEXT: ret
+ %t1 = call i8 @llvm.vector.reduce.smax.v8i8(<8 x i8> %a)
+ %t2 = insertelement <8 x i8> zeroinitializer, i8 %t1, i64 0
+ ret <8 x i8> %t2
+}
+
+define <8 x i8> @reduce_smin_v8i8_zero_fill(<8 x i8> %a) {
+; CHECK-LABEL: reduce_smin_v8i8_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sminv b0, v0.8b
+; CHECK-NEXT: ret
+ %t1 = call i8 @llvm.vector.reduce.smin.v8i8(<8 x i8> %a)
+ %t2 = insertelement <8 x i8> zeroinitializer, i8 %t1, i64 0
+ ret <8 x i8> %t2
+}
+
+define <8 x i8> @reduce_umax_v8i8_zero_fill(<8 x i8> %a) {
+; CHECK-LABEL: reduce_umax_v8i8_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umaxv b0, v0.8b
+; CHECK-NEXT: ret
+ %t1 = call i8 @llvm.vector.reduce.umax.v8i8(<8 x i8> %a)
+ %t2 = insertelement <8 x i8> zeroinitializer, i8 %t1, i64 0
+ ret <8 x i8> %t2
+}
+
+define <8 x i8> @reduce_umin_v8i8_zero_fill(<8 x i8> %a) {
+; CHECK-LABEL: reduce_umin_v8i8_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uminv b0, v0.8b
+; CHECK-NEXT: ret
+ %t1 = call i8 @llvm.vector.reduce.umin.v8i8(<8 x i8> %a)
+ %t2 = insertelement <8 x i8> zeroinitializer, i8 %t1, i64 0
+ ret <8 x i8> %t2
+}
+
+define <16 x i8> @reduce_add_v16i8_zero_fill(<16 x i8> %a) {
+; CHECK-LABEL: reduce_add_v16i8_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: addv b0, v0.16b
+; CHECK-NEXT: ret
+ %t1 = call i8 @llvm.vector.reduce.add.v16i8(<16 x i8> %a)
+ %t2 = insertelement <16 x i8> zeroinitializer, i8 %t1, i64 0
+ ret <16 x i8> %t2
+}
+
+define <4 x i16> @reduce_add_v4i16_zero_fill(<4 x i16> %a) {
+; CHECK-LABEL: reduce_add_v4i16_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: addv h0, v0.4h
+; CHECK-NEXT: ret
+ %t1 = call i16 @llvm.vector.reduce.add.v4i16(<4 x i16> %a)
+ %t2 = insertelement <4 x i16> zeroinitializer, i16 %t1, i64 0
+ ret <4 x i16> %t2
+}
+
+define <4 x i16> @reduce_smax_v4i16_zero_fill(<4 x i16> %a) {
+; CHECK-LABEL: reduce_smax_v4i16_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smaxv h0, v0.4h
+; CHECK-NEXT: ret
+ %t1 = call i16 @llvm.vector.reduce.smax.v4i16(<4 x i16> %a)
+ %t2 = insertelement <4 x i16> zeroinitializer, i16 %t1, i64 0
+ ret <4 x i16> %t2
+}
+
+define <4 x i16> @reduce_smin_v4i16_zero_fill(<4 x i16> %a) {
+; CHECK-LABEL: reduce_smin_v4i16_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sminv h0, v0.4h
+; CHECK-NEXT: ret
+ %t1 = call i16 @llvm.vector.reduce.smin.v4i16(<4 x i16> %a)
+ %t2 = insertelement <4 x i16> zeroinitializer, i16 %t1, i64 0
+ ret <4 x i16> %t2
+}
+
+define <4 x i16> @reduce_umax_v4i16_zero_fill(<4 x i16> %a) {
+; CHECK-LABEL: reduce_umax_v4i16_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umaxv h0, v0.4h
+; CHECK-NEXT: ret
+ %t1 = call i16 @llvm.vector.reduce.umax.v4i16(<4 x i16> %a)
+ %t2 = insertelement <4 x i16> zeroinitializer, i16 %t1, i64 0
+ ret <4 x i16> %t2
+}
+
+define <4 x i16> @reduce_umin_v4i16_zero_fill(<4 x i16> %a) {
+; CHECK-LABEL: reduce_umin_v4i16_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uminv h0, v0.4h
+; CHECK-NEXT: ret
+ %t1 = call i16 @llvm.vector.reduce.umin.v4i16(<4 x i16> %a)
+ %t2 = insertelement <4 x i16> zeroinitializer, i16 %t1, i64 0
+ ret <4 x i16> %t2
+}
+
+define <8 x i16> @reduce_add_v8i16_zero_fill(<8 x i16> %a) {
+; CHECK-LABEL: reduce_add_v8i16_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: addv h0, v0.8h
+; CHECK-NEXT: ret
+ %t1 = call i16 @llvm.vector.reduce.add.v8i16(<8 x i16> %a)
+ %t2 = insertelement <8 x i16> zeroinitializer, i16 %t1, i64 0
+ ret <8 x i16> %t2
+}
+
+define <4 x i32> @reduce_add_v4i32_zero_fill(<4 x i32> %a) {
+; CHECK-LABEL: reduce_add_v4i32_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: addv s0, v0.4s
+; CHECK-NEXT: ret
+ %t1 = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> %a)
+ %t2 = insertelement <4 x i32> zeroinitializer, i32 %t1, i64 0
+ ret <4 x i32> %t2
+}
+
+define <16 x i8> @reduce_smax_v16i8_zero_fill(<16 x i8> %a) {
+; CHECK-LABEL: reduce_smax_v16i8_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smaxv b0, v0.16b
+; CHECK-NEXT: ret
+ %t1 = call i8 @llvm.vector.reduce.smax.v16i8(<16 x i8> %a)
+ %t2 = insertelement <16 x i8> zeroinitializer, i8 %t1, i64 0
+ ret <16 x i8> %t2
+}
+
+define <8 x i16> @reduce_smax_v8i16_zero_fill(<8 x i16> %a) {
+; CHECK-LABEL: reduce_smax_v8i16_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smaxv h0, v0.8h
+; CHECK-NEXT: ret
+ %t1 = call i16 @llvm.vector.reduce.smax.v8i16(<8 x i16> %a)
+ %t2 = insertelement <8 x i16> zeroinitializer, i16 %t1, i64 0
+ ret <8 x i16> %t2
+}
+
+define <4 x i32> @reduce_smax_v4i32_zero_fill(<4 x i32> %a) {
+; CHECK-LABEL: reduce_smax_v4i32_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: smaxv s0, v0.4s
+; CHECK-NEXT: ret
+ %t1 = call i32 @llvm.vector.reduce.smax.v4i32(<4 x i32> %a)
+ %t2 = insertelement <4 x i32> zeroinitializer, i32 %t1, i64 0
+ ret <4 x i32> %t2
+}
+
+define <16 x i8> @reduce_smin_v16i8_zero_fill(<16 x i8> %a) {
+; CHECK-LABEL: reduce_smin_v16i8_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sminv b0, v0.16b
+; CHECK-NEXT: ret
+ %t1 = call i8 @llvm.vector.reduce.smin.v16i8(<16 x i8> %a)
+ %t2 = insertelement <16 x i8> zeroinitializer, i8 %t1, i64 0
+ ret <16 x i8> %t2
+}
+
+define <8 x i16> @reduce_smin_v8i16_zero_fill(<8 x i16> %a) {
+; CHECK-LABEL: reduce_smin_v8i16_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sminv h0, v0.8h
+; CHECK-NEXT: ret
+ %t1 = call i16 @llvm.vector.reduce.smin.v8i16(<8 x i16> %a)
+ %t2 = insertelement <8 x i16> zeroinitializer, i16 %t1, i64 0
+ ret <8 x i16> %t2
+}
+
+define <4 x i32> @reduce_smin_v4i32_zero_fill(<4 x i32> %a) {
+; CHECK-LABEL: reduce_smin_v4i32_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: sminv s0, v0.4s
+; CHECK-NEXT: ret
+ %t1 = call i32 @llvm.vector.reduce.smin.v4i32(<4 x i32> %a)
+ %t2 = insertelement <4 x i32> zeroinitializer, i32 %t1, i64 0
+ ret <4 x i32> %t2
+}
+
+define <16 x i8> @reduce_umax_v16i8_zero_fill(<16 x i8> %a) {
+; CHECK-LABEL: reduce_umax_v16i8_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umaxv b0, v0.16b
+; CHECK-NEXT: ret
+ %t1 = call i8 @llvm.vector.reduce.umax.v16i8(<16 x i8> %a)
+ %t2 = insertelement <16 x i8> zeroinitializer, i8 %t1, i64 0
+ ret <16 x i8> %t2
+}
+
+define <8 x i16> @reduce_umax_v8i16_zero_fill(<8 x i16> %a) {
+; CHECK-LABEL: reduce_umax_v8i16_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umaxv h0, v0.8h
+; CHECK-NEXT: ret
+ %t1 = call i16 @llvm.vector.reduce.umax.v8i16(<8 x i16> %a)
+ %t2 = insertelement <8 x i16> zeroinitializer, i16 %t1, i64 0
+ ret <8 x i16> %t2
+}
+
+define <4 x i32> @reduce_umax_v4i32_zero_fill(<4 x i32> %a) {
+; CHECK-LABEL: reduce_umax_v4i32_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: umaxv s0, v0.4s
+; CHECK-NEXT: ret
+ %t1 = call i32 @llvm.vector.reduce.umax.v4i32(<4 x i32> %a)
+ %t2 = insertelement <4 x i32> zeroinitializer, i32 %t1, i64 0
+ ret <4 x i32> %t2
+}
+
+define <16 x i8> @reduce_umin_v16i8_zero_fill(<16 x i8> %a) {
+; CHECK-LABEL: reduce_umin_v16i8_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uminv b0, v0.16b
+; CHECK-NEXT: ret
+ %t1 = call i8 @llvm.vector.reduce.umin.v16i8(<16 x i8> %a)
+ %t2 = insertelement <16 x i8> zeroinitializer, i8 %t1, i64 0
+ ret <16 x i8> %t2
+}
+
+define <8 x i16> @reduce_umin_v8i16_zero_fill(<8 x i16> %a) {
+; CHECK-LABEL: reduce_umin_v8i16_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uminv h0, v0.8h
+; CHECK-NEXT: ret
+ %t1 = call i16 @llvm.vector.reduce.umin.v8i16(<8 x i16> %a)
+ %t2 = insertelement <8 x i16> zeroinitializer, i16 %t1, i64 0
+ ret <8 x i16> %t2
+}
+
+define <4 x i32> @reduce_umin_v4i32_zero_fill(<4 x i32> %a) {
+; CHECK-LABEL: reduce_umin_v4i32_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: uminv s0, v0.4s
+; CHECK-NEXT: ret
+ %t1 = call i32 @llvm.vector.reduce.umin.v4i32(<4 x i32> %a)
+ %t2 = insertelement <4 x i32> zeroinitializer, i32 %t1, i64 0
+ ret <4 x i32> %t2
+}
+
+define <4 x float> @reduce_fadd_v4f32_zero_fill(<4 x float> %a) {
+; CHECK-LABEL: reduce_fadd_v4f32_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: faddp v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: faddp s0, v0.2s
+; CHECK-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-NEXT: mov v0.16b, v1.16b
+; CHECK-NEXT: ret
+ %t1 = call fast float @llvm.vector.reduce.fadd.v4f32(float -0.0, <4 x float> %a)
+ %t2 = insertelement <4 x float> zeroinitializer, float %t1, i64 0
+ ret <4 x float> %t2
+}
+
+define <4 x float> @reduce_fmaximum_v4f32_zero_fill(<4 x float> %a) {
+; CHECK-LABEL: reduce_fmaximum_v4f32_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: fmaxv s0, v0.4s
+; CHECK-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-NEXT: mov v0.16b, v1.16b
+; CHECK-NEXT: ret
+ %t1 = call float @llvm.vector.reduce.fmaximum.v4f32(<4 x float> %a)
+ %t2 = insertelement <4 x float> zeroinitializer, float %t1, i64 0
+ ret <4 x float> %t2
+}
+
+define <4 x float> @reduce_fminimum_v4f32_zero_fill(<4 x float> %a) {
+; CHECK-LABEL: reduce_fminimum_v4f32_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: fminv s0, v0.4s
+; CHECK-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-NEXT: mov v0.16b, v1.16b
+; CHECK-NEXT: ret
+ %t1 = call float @llvm.vector.reduce.fminimum.v4f32(<4 x float> %a)
+ %t2 = insertelement <4 x float> zeroinitializer, float %t1, i64 0
+ ret <4 x float> %t2
+}
+
+define <4 x float> @reduce_fmax_v4f32_zero_fill(<4 x float> %a) {
+; CHECK-LABEL: reduce_fmax_v4f32_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: fmaxnmv s0, v0.4s
+; CHECK-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-NEXT: mov v0.16b, v1.16b
+; CHECK-NEXT: ret
+ %t1 = call float @llvm.vector.reduce.fmax.v4f32(<4 x float> %a)
+ %t2 = insertelement <4 x float> zeroinitializer, float %t1, i64 0
+ ret <4 x float> %t2
+}
+
+define <4 x float> @reduce_fmin_v4f32_zero_fill(<4 x float> %a) {
+; CHECK-LABEL: reduce_fmin_v4f32_zero_fill:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v1.2d, #0000000000000000
+; CHECK-NEXT: fminnmv s0, v0.4s
+; CHECK-NEXT: mov v1.s[0], v0.s[0]
+; CHECK-NEXT: mov v0.16b, v1.16b
+; CHECK-NEXT: ret
+ %t1 = call float @llvm.vector.reduce.fmin.v4f32(<4 x float> %a)
+ %t2 = insertelement <4 x float> zeroinitializer, float %t1, i64 0
+ ret <4 x float> %t2
+}
More information about the llvm-commits
mailing list