[llvm] 5796599 - [SLP]Allow unordered fadd reductions with reassoc only
via llvm-commits
llvm-commits at lists.llvm.org
Mon Aug 3 10:50:31 PDT 2026
Author: Alexey Bataev
Date: 2026-08-03T13:50:26-04:00
New Revision: 57965996196ca9caf62784392caf478a9d4133b3
URL: https://github.com/llvm/llvm-project/commit/57965996196ca9caf62784392caf478a9d4133b3
DIFF: https://github.com/llvm/llvm-project/commit/57965996196ca9caf62784392caf478a9d4133b3.diff
LOG: [SLP]Allow unordered fadd reductions with reassoc only
The unordered reduction matcher required isAssociative() for fadd,
i.e. reassoc + nsz, a condition inherited from InstCombine-style
reassociation that also cancels and folds terms. Pure regrouping of
additions cannot change the sign of a zero result, so nsz is not
needed here: the reduction is seeded with the exact -0.0 identity,
the repeated-value multiplier preserves the sign of zero, and
constant folding is IEEE-exact. Brings the fadd requirement in line
with RecurrenceDescriptor, LV and with fmul, which accept reassoc
alone.
Reviewers: hiraditya, bababuck, RKSimon
Pull Request: https://github.com/llvm/llvm-project/pull/213261
Added:
Modified:
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
llvm/test/Transforms/PhaseOrdering/fast-basictest.ll
llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
Removed:
################################################################################
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index bf6862e68ef9f..4bf161d0f9c0d 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -29565,7 +29565,9 @@ class HorizontalReduction {
if (Kind == RecurKind::FMaximum || Kind == RecurKind::FMinimum)
return ReductionOrdering::Unordered;
- if (I->isAssociative())
+ // Reassociation alone cannot change the sign of a zero result, so nsz
+ // is not required for fadd reductions.
+ if (I->isAssociative() || (Kind == RecurKind::FAdd && I->hasAllowReassoc()))
return ReductionOrdering::Unordered;
return isCommutative(I) ? ReductionOrdering::Ordered
diff --git a/llvm/test/Transforms/PhaseOrdering/fast-basictest.ll b/llvm/test/Transforms/PhaseOrdering/fast-basictest.ll
index ec217a9cd31c6..ea62c877ceab8 100644
--- a/llvm/test/Transforms/PhaseOrdering/fast-basictest.ll
+++ b/llvm/test/Transforms/PhaseOrdering/fast-basictest.ll
@@ -45,11 +45,16 @@ define float @test2_no_FMF(float %reg109, float %reg1111) {
}
define float @test2_reassoc(float %reg109, float %reg1111) {
-; CHECK-LABEL: @test2_reassoc(
-; CHECK-NEXT: [[REG115:%.*]] = fadd reassoc float [[REG109:%.*]], -3.000000e+01
-; CHECK-NEXT: [[REG116:%.*]] = fadd reassoc float [[REG115]], [[REG1111:%.*]]
-; CHECK-NEXT: [[REG117:%.*]] = fadd reassoc float [[REG116]], 3.000000e+01
-; CHECK-NEXT: ret float [[REG117]]
+; REASSOC_AND_IC-LABEL: @test2_reassoc(
+; REASSOC_AND_IC-NEXT: [[REG115:%.*]] = fadd reassoc float [[REG109:%.*]], -3.000000e+01
+; REASSOC_AND_IC-NEXT: [[REG116:%.*]] = fadd reassoc float [[REG115]], [[REG1111:%.*]]
+; REASSOC_AND_IC-NEXT: [[REG117:%.*]] = fadd reassoc float [[REG116]], 3.000000e+01
+; REASSOC_AND_IC-NEXT: ret float [[REG117]]
+;
+; O2-LABEL: @test2_reassoc(
+; O2-NEXT: [[OP_RDX:%.*]] = fadd reassoc float [[REG109:%.*]], 0.000000e+00
+; O2-NEXT: [[OP_RDX1:%.*]] = fadd reassoc float [[OP_RDX]], [[REG1111:%.*]]
+; O2-NEXT: ret float [[OP_RDX1]]
;
%reg115 = fadd reassoc float %reg109, -3.000000e+01
%reg116 = fadd reassoc float %reg115, %reg1111
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll b/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
index d095ee31652f7..7b6c896154c7e 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
@@ -6,10 +6,8 @@ define double @test(double %0) {
; CHECK-SAME: double [[TMP0:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[TMP5:%.*]] = fmul double [[TMP0]], 0.000000e+00
-; CHECK-NEXT: [[TMP4:%.*]] = fadd reassoc double [[TMP5]], 0.000000e+00
-; CHECK-NEXT: [[OP_RDX:%.*]] = fadd reassoc double [[TMP5]], [[TMP4]]
-; CHECK-NEXT: [[TMP6:%.*]] = fadd reassoc double [[TMP5]], [[OP_RDX]]
-; CHECK-NEXT: [[TMP7:%.*]] = fadd reassoc double [[TMP5]], [[TMP6]]
+; CHECK-NEXT: [[TMP2:%.*]] = fmul double [[TMP5]], 4.000000e+00
+; CHECK-NEXT: [[TMP7:%.*]] = fadd reassoc double [[TMP2]], 0.000000e+00
; CHECK-NEXT: ret double [[TMP7]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
index 0c629cec2c827..9cd95683d0264 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-fadd-reassoc.ll
@@ -7,28 +7,8 @@ define double @reassoc_only(ptr %p) {
; CHECK-LABEL: define double @reassoc_only(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[L0:%.*]] = load double, ptr [[P]], align 8
-; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P]], i64 1
-; CHECK-NEXT: [[L1:%.*]] = load double, ptr [[P1]], align 8
-; CHECK-NEXT: [[P2:%.*]] = getelementptr double, ptr [[P]], i64 2
-; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
-; CHECK-NEXT: [[P3:%.*]] = getelementptr double, ptr [[P]], i64 3
-; CHECK-NEXT: [[L3:%.*]] = load double, ptr [[P3]], align 8
-; CHECK-NEXT: [[P4:%.*]] = getelementptr double, ptr [[P]], i64 4
-; CHECK-NEXT: [[L4:%.*]] = load double, ptr [[P4]], align 8
-; CHECK-NEXT: [[P5:%.*]] = getelementptr double, ptr [[P]], i64 5
-; CHECK-NEXT: [[L5:%.*]] = load double, ptr [[P5]], align 8
-; CHECK-NEXT: [[P6:%.*]] = getelementptr double, ptr [[P]], i64 6
-; CHECK-NEXT: [[L6:%.*]] = load double, ptr [[P6]], align 8
-; CHECK-NEXT: [[P7:%.*]] = getelementptr double, ptr [[P]], i64 7
-; CHECK-NEXT: [[L7:%.*]] = load double, ptr [[P7]], align 8
-; CHECK-NEXT: [[C1:%.*]] = fadd reassoc double [[L0]], [[L1]]
-; CHECK-NEXT: [[C2:%.*]] = fadd reassoc double [[C1]], [[L2]]
-; CHECK-NEXT: [[C3:%.*]] = fadd reassoc double [[C2]], [[L3]]
-; CHECK-NEXT: [[C4:%.*]] = fadd reassoc double [[C3]], [[L4]]
-; CHECK-NEXT: [[C5:%.*]] = fadd reassoc double [[C4]], [[L5]]
-; CHECK-NEXT: [[C6:%.*]] = fadd reassoc double [[C5]], [[L6]]
-; CHECK-NEXT: [[TMP1:%.*]] = fadd reassoc double [[C6]], [[L7]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <8 x double>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc double @llvm.vector.reduce.fadd.v8f64(double -0.000000e+00, <8 x double> [[TMP0]])
; CHECK-NEXT: ret double [[TMP1]]
;
entry:
@@ -63,14 +43,7 @@ define double @reassoc_repeated_vals(double %x) {
; CHECK-LABEL: define double @reassoc_repeated_vals(
; CHECK-SAME: double [[X:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[C1:%.*]] = fadd reassoc double [[X]], [[X]]
-; CHECK-NEXT: [[C2:%.*]] = fadd reassoc double [[C1]], [[X]]
-; CHECK-NEXT: [[C3:%.*]] = fadd reassoc double [[C2]], [[X]]
-; CHECK-NEXT: [[C4:%.*]] = fadd reassoc double [[C3]], [[X]]
-; CHECK-NEXT: [[C5:%.*]] = fadd reassoc double [[C4]], [[X]]
-; CHECK-NEXT: [[C6:%.*]] = fadd reassoc double [[C5]], [[X]]
-; CHECK-NEXT: [[C7:%.*]] = fadd reassoc double [[C6]], [[X]]
-; CHECK-NEXT: [[TMP0:%.*]] = fadd reassoc double [[C7]], [[X]]
+; CHECK-NEXT: [[TMP0:%.*]] = fmul double [[X]], 9.000000e+00
; CHECK-NEXT: ret double [[TMP0]]
;
entry:
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
index e12263b02103c..0ce19e482d072 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
@@ -128,14 +128,14 @@ entry:
ret float %r6
}
-define float @reduce_ordered_fadd_reassoc_no_nsz(ptr %p) {
-; CHECK-LABEL: define float @reduce_ordered_fadd_reassoc_no_nsz(
+define float @reduce_unordered_fadd_reassoc_no_nsz(ptr %p) {
+; CHECK-LABEL: define float @reduce_unordered_fadd_reassoc_no_nsz(
; CHECK-SAME: ptr [[P:%.*]]) #[[ATTR0]] {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds float, ptr [[P]], i64 0
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[P0]], align 4
-; CHECK-NEXT: [[R6:%.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[TMP0]])
-; CHECK-NEXT: ret float [[R6]]
+; CHECK-NEXT: [[TMP1:%.*]] = call reassoc float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[TMP0]])
+; CHECK-NEXT: ret float [[TMP1]]
;
entry:
%p0 = getelementptr inbounds float, ptr %p, i64 0
More information about the llvm-commits
mailing list