[llvm] [SLP]Allow unordered fadd reductions with reassoc only (PR #213261)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Fri Jul 31 05:21:44 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/213261
The unordered reduction matcher required isAssociative() for fadd,
i.e. reassoc + nsz, a condition inherited from InstCombine-style
reassociation that also cancels and folds terms. Pure regrouping of
additions cannot change the sign of a zero result, so nsz is not
needed here: the reduction is seeded with the exact -0.0 identity,
the repeated-value multiplier preserves the sign of zero, and
constant folding is IEEE-exact. Brings the fadd requirement in line
with RecurrenceDescriptor, LV and with fmul, which accept reassoc
alone.
>From b3825e9941472bc7bc957eb44631e38b927e43d9 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Fri, 31 Jul 2026 05:21:26 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../Transforms/Vectorize/SLPVectorizer.cpp | 4 ++-
.../PhaseOrdering/fast-basictest.ll | 15 ++++++---
.../X86/ordered-reduction-replaced.ll | 6 ++--
.../X86/reduction-fadd-reassoc.ll | 33 ++-----------------
.../X86/reduction-ordered-fadd.ll | 4 +--
5 files changed, 20 insertions(+), 42 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 069facf6fb4e0..c1779841d1c15 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29529,7 +29529,9 @@ class HorizontalReduction {
if (Kind == RecurKind::FMaximum || Kind == RecurKind::FMinimum)
return ReductionOrdering::Unordered;
- if (I->isAssociative())
+ // Reassociation alone cannot change the sign of a zero result, so nsz
+ // is not required for fadd reductions.
+ if (I->isAssociative() || (Kind == RecurKind::FAdd && I->hasAllowReassoc()))
return ReductionOrdering::Unordered;
return isCommutative(I) ? ReductionOrdering::Ordered
diff --git a/llvm/test/Transforms/PhaseOrdering/fast-basictest.ll b/llvm/test/Transforms/PhaseOrdering/fast-basictest.ll
index ec217a9cd31c6..ea62c877ceab8 100644
--- a/llvm/test/Transforms/PhaseOrdering/fast-basictest.ll
+++ b/llvm/test/Transforms/PhaseOrdering/fast-basictest.ll
@@ -45,11 +45,16 @@ define float @test2_no_FMF(float %reg109, float %reg1111) {
}
define float @test2_reassoc(float %reg109, float %reg1111) {
-; CHECK-LABEL: @test2_reassoc(
-; CHECK-NEXT: [[REG115:%.*]] = fadd reassoc float [[REG109:%.*]], -3.000000e+01
-; CHECK-NEXT: [[REG116:%.*]] = fadd reassoc float [[REG115]], [[REG1111:%.*]]
-; CHECK-NEXT: [[REG117:%.*]] = fadd reassoc float [[REG116]], 3.000000e+01
-; CHECK-NEXT: ret float [[REG117]]
+; REASSOC_AND_IC-LABEL: @test2_reassoc(
+; REASSOC_AND_IC-NEXT: [[REG115:%.*]] = fadd reassoc float [[REG109:%.*]], -3.000000e+01
+; REASSOC_AND_IC-NEXT: [[REG116:%.*]] = fadd reassoc float [[REG115]], [[REG1111:%.*]]
+; REASSOC_AND_IC-NEXT: [[REG117:%.*]] = fadd reassoc float [[REG116]], 3.000000e+01
+; REASSOC_AND_IC-NEXT: ret float [[REG117]]
+;
+; O2-LABEL: @test2_reassoc(
+; O2-NEXT: [[OP_RDX:%.*]] = fadd reassoc float [[REG109:%.*]], 0.000000e+00
+; O2-NEXT: [[OP_RDX1:%.*]] = fadd reassoc float [[OP_RDX]], [[REG1111:%.*]]
+; O2-NEXT: ret float [[OP_RDX1]]
;
%reg115 = fadd reassoc float %reg109, -3.000000e+01
%reg116 = fadd reassoc float %reg115, %reg1111
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll b/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
index d095ee31652f7..7b6c896154c7e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
@@ -6,10 +6,8 @@ define double @test(double %0) {
; CHECK-SAME: double [[TMP0:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP5:%.*]] = fmul double [[TMP0]], 0.000000e+00
-; CHECK-NEXT: [[TMP4:%.*]] = fadd reassoc double [[TMP5]], 0.000000e+00
-; CHECK-NEXT: [[OP_RDX:%.*]] = fadd reassoc double [[TMP5]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = fadd reassoc double [[TMP5]], [[OP_RDX]]
-; CHECK-NEXT: [[TMP7:%.*]] = fadd reassoc double [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP2:%.*]] = fmul double [[TMP5]], 4.000000e+00
+; CHECK-NEXT: [[TMP7:%.*]] = fadd reassoc double [[TMP2]], 0.000000e+00
; CHECK-NEXT: ret double [[TMP7]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
index 0c629cec2c827..9cd95683d0264 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
@@ -7,28 +7,8 @@ define double @reassoc_only(ptr %p) {
; CHECK-LABEL: define double @reassoc_only(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P]], align 8
-; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P]], i64 1
-; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[P2:%.*]] = getelementptr double, ptr [[P]], i64 2
-; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[P3:%.*]] = getelementptr double, ptr [[P]], i64 3
-; CHECK-NEXT: [[L3:%.*]] = load double, ptr [[P3]], align 8
-; CHECK-NEXT: [[P4:%.*]] = getelementptr double, ptr [[P]], i64 4
-; CHECK-NEXT: [[L4:%.*]] = load double, ptr [[P4]], align 8
-; CHECK-NEXT: [[P5:%.*]] = getelementptr double, ptr [[P]], i64 5
-; CHECK-NEXT: [[L5:%.*]] = load double, ptr [[P5]], align 8
-; CHECK-NEXT: [[P6:%.*]] = getelementptr double, ptr [[P]], i64 6
-; CHECK-NEXT: [[L6:%.*]] = load double, ptr [[P6]], align 8
-; CHECK-NEXT: [[P7:%.*]] = getelementptr double, ptr [[P]], i64 7
-; CHECK-NEXT: [[L7:%.*]] = load double, ptr [[P7]], align 8
-; CHECK-NEXT: [[C1:%.*]] = fadd reassoc double [[L0]], [[L1]]
-; CHECK-NEXT: [[C2:%.*]] = fadd reassoc double [[C1]], [[L2]]
-; CHECK-NEXT: [[C3:%.*]] = fadd reassoc double [[C2]], [[L3]]
-; CHECK-NEXT: [[C4:%.*]] = fadd reassoc double [[C3]], [[L4]]
-; CHECK-NEXT: [[C5:%.*]] = fadd reassoc double [[C4]], [[L5]]
-; CHECK-NEXT: [[C6:%.*]] = fadd reassoc double [[C5]], [[L6]]
-; CHECK-NEXT: [[TMP1:%.*]] = fadd reassoc double [[C6]], [[L7]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x double>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc double @llvm.vector.reduce.fadd.v8f64(double -0.000000e+00, <8 x double> [[TMP0]])
; CHECK-NEXT: ret double [[TMP1]]
;
entry:
@@ -63,14 +43,7 @@ define double @reassoc_repeated_vals(double %x) {
; CHECK-LABEL: define double @reassoc_repeated_vals(
; CHECK-SAME: double [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[C1:%.*]] = fadd reassoc double [[X]], [[X]]
-; CHECK-NEXT: [[C2:%.*]] = fadd reassoc double [[C1]], [[X]]
-; CHECK-NEXT: [[C3:%.*]] = fadd reassoc double [[C2]], [[X]]
-; CHECK-NEXT: [[C4:%.*]] = fadd reassoc double [[C3]], [[X]]
-; CHECK-NEXT: [[C5:%.*]] = fadd reassoc double [[C4]], [[X]]
-; CHECK-NEXT: [[C6:%.*]] = fadd reassoc double [[C5]], [[X]]
-; CHECK-NEXT: [[C7:%.*]] = fadd reassoc double [[C6]], [[X]]
-; CHECK-NEXT: [[TMP0:%.*]] = fadd reassoc double [[C7]], [[X]]
+; CHECK-NEXT: [[TMP0:%.*]] = fmul double [[X]], 9.000000e+00
; CHECK-NEXT: ret double [[TMP0]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
index e12263b02103c..17961ae719e7f 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
@@ -128,13 +128,13 @@ entry:
ret float %r6
}
-define float @reduce_ordered_fadd_reassoc_no_nsz(ptr %p) {
+define float @reduce_unordered_fadd_reassoc_no_nsz(ptr %p) {
; CHECK-LABEL: define float @reduce_ordered_fadd_reassoc_no_nsz(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds float, ptr [[P]], i64 0
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[P0]], align 4
-; CHECK-NEXT: [[R6:%.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[TMP0]])
+; CHECK-NEXT: [[R6:%.*]] = call reassoc float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[TMP0]])
; CHECK-NEXT: ret float [[R6]]
;
entry:
More information about the llvm-commits
mailing list