[llvm] [AArch64] Decompose FADDV with known zero elements (PR #167313)
Guy David via llvm-commits
llvm-commits at lists.llvm.org
Sun Feb 15 07:17:35 PST 2026
https://github.com/guy-david updated https://github.com/llvm/llvm-project/pull/167313
>From ca9d07e7aee7499d540107bdf59cbb5f58815482 Mon Sep 17 00:00:00 2001
From: Guy David <guyda96 at gmail.com>
Date: Sun, 9 Nov 2025 17:55:53 +0200
Subject: [PATCH] [AArch64] Decompose FADD reductions with known zero elements
FADDV is matched into FADDPv4f32 + FADDPv2f32p but this can be relaxed
when one element (usually the 4th) or more are known to be zero.
Before:
movi d1, #0000000000000000
mov v0.s[3], v1.s[0]
faddp v0.4s, v0.4s, v0.4s
faddp s0, v0.2s
After:
mov s1, v0.s[2]
faddp s0, v0.2s
fadd s0, s0, s1
---
.../Target/AArch64/AArch64ISelLowering.cpp | 101 +++++++
llvm/test/CodeGen/AArch64/faddv.ll | 256 ++++++++++++++++++
2 files changed, 357 insertions(+)
create mode 100644 llvm/test/CodeGen/AArch64/faddv.ll
diff --git a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
index 66c22db0491d1..2ad651baec54a 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelLowering.cpp
@@ -1188,6 +1188,8 @@ AArch64TargetLowering::AArch64TargetLowering(const TargetMachine &TM,
setTargetDAGCombine(ISD::VECREDUCE_AND);
setTargetDAGCombine(ISD::VECREDUCE_OR);
setTargetDAGCombine(ISD::VECREDUCE_XOR);
+ setTargetDAGCombine(ISD::VECREDUCE_FADD);
+ setTargetDAGCombine(ISD::VECREDUCE_SEQ_FADD);
setTargetDAGCombine(ISD::SCALAR_TO_VECTOR);
@@ -23693,6 +23695,98 @@ static SDValue combineSVEBitSel(unsigned IID, SDNode *N, SelectionDAG &DAG) {
}
}
+/// Optimize patterns where we insert zeros into vector lanes before a
+/// floating-point add reduction. This handles both the aarch64.neon.faddv
+/// intrinsic and ISD::VECREDUCE_FADD/VECREDUCE_SEQ_FADD.
+/// When Start is set, the reduction is sequential (left-to-right), otherwise
+/// it is pairwise (tree).
+static SDValue tryCombineFADDReductionWithZero(SDNode *N, SelectionDAG &DAG,
+ SDValue Vec,
+ SDValue Start = SDValue()) {
+ EVT VT = Vec.getValueType();
+ // Don't handle scalable vectors, they don't fit the NEON FADDP pattern.
+ if (!VT.isFixedLengthVector())
+ return SDValue();
+ EVT EltVT = VT.getVectorElementType();
+ // Don't handle f16, scalar f16 FADD requires promotion to f32 which causes
+ // redundant round-trip conversions between each FADD.
+ if (EltVT != MVT::f32 && EltVT != MVT::f64)
+ return SDValue();
+
+ SDLoc DL(N);
+ unsigned NumElts = VT.getVectorNumElements();
+ // Pairwise reduction requires power-of-2 number of elements.
+ if (!isPowerOf2_32(NumElts))
+ return SDValue();
+ APInt DemandedElts = APInt::getAllOnes(NumElts);
+ APInt KnownZeroElts = DAG.computeVectorKnownZeroElements(Vec, DemandedElts);
+ unsigned NumZeroElts = KnownZeroElts.popcount();
+ // No element is known to be +0.0, fallback to the TableGen pattern.
+ if (NumZeroElts == 0)
+ return SDValue();
+ // All elements are +0.0.
+ if (NumZeroElts == NumElts)
+ return Start ? Start : DAG.getConstantFP(0.0, DL, EltVT);
+
+ // At least one element is +0.0, so it is worth to decompose the reduction
+ // into fadd's. FADDV is a pairwise reduction, so we need to respect the
+ // order of the elements in the vector.
+
+ // Check if we can output a signed zero.
+ // This avoids the scenario where all the added values are -0.0 except the
+ // +0.0 element we chose to ignore.
+ SDNodeFlags Flags = N->getFlags();
+ bool IsSignedZeroSafe =
+ Flags.hasNoSignedZeros() || DAG.canIgnoreSignBitOfZero(SDValue(N, 0));
+ if (!IsSignedZeroSafe)
+ return SDValue();
+
+ Flags.setNoSignedZeros(true);
+ // Sequential reduction- add every element in-order, skipping zeros.
+ if (Start) {
+ SDValue Acc = Start;
+ for (unsigned I = 0; I < NumElts; I++) {
+ if (KnownZeroElts[I])
+ continue;
+ SDValue Elt = DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
+ DAG.getConstant(I, DL, MVT::i64));
+
+ Acc = DAG.getNode(ISD::FADD, DL, EltVT, Acc, Elt, Flags);
+ }
+ return Acc;
+ }
+
+ // Pairwise reduction- extract all elements, then reduce pairwise.
+ SmallVector<SDValue, 4> Elts;
+ for (unsigned I = 0; I < NumElts; I++) {
+ Elts.push_back(DAG.getNode(ISD::EXTRACT_VECTOR_ELT, DL, EltVT, Vec,
+ DAG.getConstant(I, DL, MVT::i64)));
+ }
+ while (Elts.size() > 1) {
+ SmallVector<SDValue, 4> NewElts;
+ APInt NewKnownZeroElts = APInt::getZero(Elts.size() / 2);
+ for (unsigned I = 0; I < Elts.size(); I += 2) {
+ bool ZeroI = KnownZeroElts[I];
+ bool ZeroI1 = KnownZeroElts[I + 1];
+ if (ZeroI && ZeroI1) {
+ // Both elements are zero, result is zero.
+ NewElts.push_back(Elts[I]);
+ NewKnownZeroElts.setBit(I / 2);
+ } else if (ZeroI) {
+ NewElts.push_back(Elts[I + 1]);
+ } else if (ZeroI1) {
+ NewElts.push_back(Elts[I]);
+ } else {
+ NewElts.push_back(
+ DAG.getNode(ISD::FADD, DL, EltVT, Elts[I], Elts[I + 1], Flags));
+ }
+ }
+ Elts = std::move(NewElts);
+ KnownZeroElts = NewKnownZeroElts;
+ }
+ return Elts[0];
+}
+
static SDValue performIntrinsicCombine(SDNode *N,
TargetLowering::DAGCombinerInfo &DCI,
const AArch64Subtarget *Subtarget) {
@@ -23716,6 +23810,8 @@ static SDValue performIntrinsicCombine(SDNode *N,
return combineAcrossLanesIntrinsic(AArch64ISD::SMAXV, N, DAG);
case Intrinsic::aarch64_neon_umaxv:
return combineAcrossLanesIntrinsic(AArch64ISD::UMAXV, N, DAG);
+ case Intrinsic::aarch64_neon_faddv:
+ return tryCombineFADDReductionWithZero(N, DAG, N->getOperand(1));
case Intrinsic::aarch64_neon_fmax:
return DAG.getNode(ISD::FMAXIMUM, SDLoc(N), N->getValueType(0),
N->getOperand(1), N->getOperand(2));
@@ -29052,6 +29148,11 @@ SDValue AArch64TargetLowering::PerformDAGCombine(SDNode *N,
return performExtractVectorEltCombine(N, DCI, Subtarget);
case ISD::VECREDUCE_ADD:
return performVecReduceAddCombine(N, DCI.DAG, Subtarget);
+ case ISD::VECREDUCE_FADD:
+ return tryCombineFADDReductionWithZero(N, DCI.DAG, N->getOperand(0));
+ case ISD::VECREDUCE_SEQ_FADD:
+ return tryCombineFADDReductionWithZero(N, DCI.DAG, N->getOperand(1),
+ N->getOperand(0));
case ISD::GET_ACTIVE_LANE_MASK:
return performActiveLaneMaskCombine(N, DCI, Subtarget);
case AArch64ISD::UADDV:
diff --git a/llvm/test/CodeGen/AArch64/faddv.ll b/llvm/test/CodeGen/AArch64/faddv.ll
new file mode 100644
index 0000000000000..5b27a565e1e19
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/faddv.ll
@@ -0,0 +1,256 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
+; RUN: llc -mtriple=aarch64 < %s | FileCheck %s
+
+define float @test_v2f32_element_0_zero(<2 x float> %vec) {
+; CHECK-LABEL: test_v2f32_element_0_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: mov s0, v0.s[1]
+; CHECK-NEXT: ret
+entry:
+ %with_zero = insertelement <2 x float> %vec, float 0.0, i64 0
+ %sum = call nsz float @llvm.aarch64.neon.faddv.f32.v2f32(<2 x float> %with_zero)
+ ret float %sum
+}
+
+define float @test_v4f32_element_3_zero(<4 x float> %vec) {
+; CHECK-LABEL: test_v4f32_element_3_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov s1, v0.s[2]
+; CHECK-NEXT: faddp s0, v0.2s
+; CHECK-NEXT: fadd s0, s0, s1
+; CHECK-NEXT: ret
+entry:
+ %with_zero = insertelement <4 x float> %vec, float 0.0, i64 3
+ %sum = call nsz float @llvm.aarch64.neon.faddv.f32.v4f32(<4 x float> %with_zero)
+ ret float %sum
+}
+
+define float @test_v4f32_elements_0_2_zero(<4 x float> %vec) {
+; CHECK-LABEL: test_v4f32_elements_0_2_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov s1, v0.s[3]
+; CHECK-NEXT: mov s0, v0.s[1]
+; CHECK-NEXT: fadd s0, s0, s1
+; CHECK-NEXT: ret
+entry:
+ %zero1 = insertelement <4 x float> %vec, float 0.0, i64 0
+ %zero2 = insertelement <4 x float> %zero1, float 0.0, i64 2
+ %sum = call nsz float @llvm.aarch64.neon.faddv.f32.v4f32(<4 x float> %zero2)
+ ret float %sum
+}
+
+define float @test_v4f32_all_zero(<4 x float> %vec) {
+; CHECK-LABEL: test_v4f32_all_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: ret
+entry:
+ %zero1 = insertelement <4 x float> %vec, float 0.0, i64 0
+ %zero2 = insertelement <4 x float> %zero1, float 0.0, i64 1
+ %zero3 = insertelement <4 x float> %zero2, float 0.0, i64 2
+ %zero4 = insertelement <4 x float> %zero3, float 0.0, i64 3
+ %sum = call float @llvm.aarch64.neon.faddv.f32.v4f32(<4 x float> %zero4)
+ ret float %sum
+}
+
+define double @test_v2f64_element_0_zero(<2 x double> %vec) {
+; CHECK-LABEL: test_v2f64_element_0_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov d0, v0.d[1]
+; CHECK-NEXT: ret
+entry:
+ %with_zero = insertelement <2 x double> %vec, double 0.0, i64 0
+ %sum = call nsz double @llvm.aarch64.neon.faddv.f64.v2f64(<2 x double> %with_zero)
+ ret double %sum
+}
+
+define float @negative_test(<4 x float> %vec) {
+; CHECK-LABEL: negative_test:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: mov v0.s[3], v1.s[0]
+; CHECK-NEXT: faddp v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: faddp s0, v0.2s
+; CHECK-NEXT: ret
+ %with_zero = insertelement <4 x float> %vec, float 0.0, i64 3
+ %sum = call float @llvm.aarch64.neon.faddv.f32.v4f32(<4 x float> %with_zero)
+ ret float %sum
+}
+
+define float @test_reduce_v2f32_element_0_zero(<2 x float> %vec) {
+; CHECK-LABEL: test_reduce_v2f32_element_0_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: mov s0, v0.s[1]
+; CHECK-NEXT: ret
+entry:
+ %with_zero = insertelement <2 x float> %vec, float 0.0, i64 0
+ %sum = call reassoc nsz float @llvm.vector.reduce.fadd.v2f32(float -0.0, <2 x float> %with_zero)
+ ret float %sum
+}
+
+define float @test_reduce_v4f32_element_3_zero(<4 x float> %vec) {
+; CHECK-LABEL: test_reduce_v4f32_element_3_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov s1, v0.s[2]
+; CHECK-NEXT: faddp s0, v0.2s
+; CHECK-NEXT: fadd s0, s0, s1
+; CHECK-NEXT: ret
+entry:
+ %with_zero = insertelement <4 x float> %vec, float 0.0, i64 3
+ %sum = call reassoc nsz float @llvm.vector.reduce.fadd.v4f32(float -0.0, <4 x float> %with_zero)
+ ret float %sum
+}
+
+define float @test_reduce_v4f32_elements_0_2_zero(<4 x float> %vec) {
+; CHECK-LABEL: test_reduce_v4f32_elements_0_2_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov s1, v0.s[3]
+; CHECK-NEXT: mov s0, v0.s[1]
+; CHECK-NEXT: fadd s0, s0, s1
+; CHECK-NEXT: ret
+entry:
+ %zero1 = insertelement <4 x float> %vec, float 0.0, i64 0
+ %zero2 = insertelement <4 x float> %zero1, float 0.0, i64 2
+ %sum = call reassoc nsz float @llvm.vector.reduce.fadd.v4f32(float -0.0, <4 x float> %zero2)
+ ret float %sum
+}
+
+define float @test_reduce_v4f32_all_zero(<4 x float> %vec) {
+; CHECK-LABEL: test_reduce_v4f32_all_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: movi d0, #0000000000000000
+; CHECK-NEXT: ret
+entry:
+ %zero1 = insertelement <4 x float> %vec, float 0.0, i64 0
+ %zero2 = insertelement <4 x float> %zero1, float 0.0, i64 1
+ %zero3 = insertelement <4 x float> %zero2, float 0.0, i64 2
+ %zero4 = insertelement <4 x float> %zero3, float 0.0, i64 3
+ %sum = call reassoc nsz float @llvm.vector.reduce.fadd.v4f32(float -0.0, <4 x float> %zero4)
+ ret float %sum
+}
+
+define double @test_reduce_v2f64_element_0_zero(<2 x double> %vec) {
+; CHECK-LABEL: test_reduce_v2f64_element_0_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov d0, v0.d[1]
+; CHECK-NEXT: ret
+entry:
+ %with_zero = insertelement <2 x double> %vec, double 0.0, i64 0
+ %sum = call reassoc nsz double @llvm.vector.reduce.fadd.v2f64(double -0.0, <2 x double> %with_zero)
+ ret double %sum
+}
+
+define float @negative_test_reduce(<4 x float> %vec) {
+; CHECK-LABEL: negative_test_reduce:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: mov v0.s[3], v1.s[0]
+; CHECK-NEXT: faddp v0.4s, v0.4s, v0.4s
+; CHECK-NEXT: faddp s0, v0.2s
+; CHECK-NEXT: ret
+ %with_zero = insertelement <4 x float> %vec, float 0.0, i64 3
+ %sum = call reassoc float @llvm.vector.reduce.fadd.v4f32(float -0.0, <4 x float> %with_zero)
+ ret float %sum
+}
+
+define float @test_seq_reduce_v2f32_element_0_zero(<2 x float> %vec) {
+; CHECK-LABEL: test_seq_reduce_v2f32_element_0_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT: mov s0, v0.s[1]
+; CHECK-NEXT: ret
+entry:
+ %with_zero = insertelement <2 x float> %vec, float 0.0, i64 0
+ %sum = call nsz float @llvm.vector.reduce.fadd.v2f32(float -0.0, <2 x float> %with_zero)
+ ret float %sum
+}
+
+define float @test_seq_reduce_v4f32_element_3_zero(<4 x float> %vec) {
+; CHECK-LABEL: test_seq_reduce_v4f32_element_3_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov s1, v0.s[2]
+; CHECK-NEXT: faddp s0, v0.2s
+; CHECK-NEXT: fadd s0, s0, s1
+; CHECK-NEXT: ret
+entry:
+ %with_zero = insertelement <4 x float> %vec, float 0.0, i64 3
+ %sum = call nsz float @llvm.vector.reduce.fadd.v4f32(float -0.0, <4 x float> %with_zero)
+ ret float %sum
+}
+
+define float @test_seq_reduce_v4f32_elements_0_2_zero(<4 x float> %vec) {
+; CHECK-LABEL: test_seq_reduce_v4f32_elements_0_2_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov s1, v0.s[3]
+; CHECK-NEXT: mov s0, v0.s[1]
+; CHECK-NEXT: fadd s0, s0, s1
+; CHECK-NEXT: ret
+entry:
+ %zero1 = insertelement <4 x float> %vec, float 0.0, i64 0
+ %zero2 = insertelement <4 x float> %zero1, float 0.0, i64 2
+ %sum = call nsz float @llvm.vector.reduce.fadd.v4f32(float -0.0, <4 x float> %zero2)
+ ret float %sum
+}
+
+define float @test_seq_reduce_v4f32_all_zero(float %start, <4 x float> %vec) {
+; CHECK-LABEL: test_seq_reduce_v4f32_all_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: ret
+entry:
+ %zero1 = insertelement <4 x float> %vec, float 0.0, i64 0
+ %zero2 = insertelement <4 x float> %zero1, float 0.0, i64 1
+ %zero3 = insertelement <4 x float> %zero2, float 0.0, i64 2
+ %zero4 = insertelement <4 x float> %zero3, float 0.0, i64 3
+ %sum = call nsz float @llvm.vector.reduce.fadd.v4f32(float %start, <4 x float> %zero4)
+ ret float %sum
+}
+
+define float @test_seq_reduce_with_start(float %start, <4 x float> %vec) {
+; CHECK-LABEL: test_seq_reduce_with_start:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fadd s0, s0, s1
+; CHECK-NEXT: mov s2, v1.s[1]
+; CHECK-NEXT: mov s1, v1.s[2]
+; CHECK-NEXT: fadd s0, s0, s2
+; CHECK-NEXT: fadd s0, s0, s1
+; CHECK-NEXT: ret
+entry:
+ %with_zero = insertelement <4 x float> %vec, float 0.0, i64 3
+ %sum = call nsz float @llvm.vector.reduce.fadd.v4f32(float %start, <4 x float> %with_zero)
+ ret float %sum
+}
+
+define double @test_seq_reduce_v2f64_element_0_zero(<2 x double> %vec) {
+; CHECK-LABEL: test_seq_reduce_v2f64_element_0_zero:
+; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: mov d0, v0.d[1]
+; CHECK-NEXT: ret
+entry:
+ %with_zero = insertelement <2 x double> %vec, double 0.0, i64 0
+ %sum = call nsz double @llvm.vector.reduce.fadd.v2f64(double -0.0, <2 x double> %with_zero)
+ ret double %sum
+}
+
+define float @negative_test_seq_reduce(<4 x float> %vec) {
+; CHECK-LABEL: negative_test_seq_reduce:
+; CHECK: // %bb.0:
+; CHECK-NEXT: mov s2, v0.s[2]
+; CHECK-NEXT: faddp s0, v0.2s
+; CHECK-NEXT: movi d1, #0000000000000000
+; CHECK-NEXT: fadd s0, s0, s2
+; CHECK-NEXT: fadd s0, s0, s1
+; CHECK-NEXT: ret
+ %with_zero = insertelement <4 x float> %vec, float 0.0, i64 3
+ %sum = call float @llvm.vector.reduce.fadd.v4f32(float -0.0, <4 x float> %with_zero)
+ ret float %sum
+}
+
+declare float @llvm.aarch64.neon.faddv.f32.v2f32(<2 x float>)
+declare float @llvm.aarch64.neon.faddv.f32.v4f32(<4 x float>)
+declare double @llvm.aarch64.neon.faddv.f64.v2f64(<2 x double>)
+
+declare float @llvm.vector.reduce.fadd.v2f32(float, <2 x float>)
+declare float @llvm.vector.reduce.fadd.v4f32(float, <4 x float>)
+declare double @llvm.vector.reduce.fadd.v2f64(double, <2 x double>)
More information about the llvm-commits
mailing list