[llvm] [SLP]Keep ordered fadd reductions sequential (PR #203741)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Sat Jun 13 19:17:38 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/203741
tryToReduceOrdered vectorizes reductions that are not associative (e.g. an
fadd with reassoc but without nsz). The accumulation order must be
preserved, but the reduction was costed and emitted with reassoc, and the
whole tree was rotated to memory order by reorderTopToBottom, which can
change the result of the sequential reduction.
Drop reassoc from the flags used to cost and emit the reduction so the
generated llvm.vector.reduce.fadd stays ordered, and drop the top-to-bottom
reorder so the reduced values keep their original accumulation order.
>From 922f9a921cfa291e508a1ae63bbbc4652169df84 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Sat, 13 Jun 2026 19:17:27 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp | 9 ++++++++-
.../SLPVectorizer/X86/ordered-reduction-replaced.ll | 11 +++++------
.../X86/ordered-reduction-root-deleted.ll | 3 ++-
.../SLPVectorizer/X86/reduction-ordered-fadd.ll | 5 +++--
4 files changed, 18 insertions(+), 10 deletions(-)
diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
index 5e58d0eb0fa26..94ddd172a9774 100644
--- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
+++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp
@@ -30266,6 +30266,12 @@ class HorizontalReduction {
for (Instruction *Op : ReducedValsToOps.at(RdxVal))
if (auto *FPMO = dyn_cast<FPMathOperator>(Op))
RdxFMF &= FPMO->getFastMathFlags();
+ // This is an ordered reduction, selected because the reduction operations
+ // are not associative (e.g. an fadd without nsz). The accumulation order
+ // must be preserved, so the generated llvm.vector.reduce.fadd has to stay
+ // ordered. Drop 'reassoc' from the flags used to emit and cost the
+ // reduction.
+ RdxFMF.setAllowReassoc(/*B=*/false);
unsigned MaxVecRegSize = V.getMaxVecRegSize();
unsigned EltSize = V.getVectorElementSize(Candidates[0]);
@@ -30327,7 +30333,8 @@ class HorizontalReduction {
V.analyzedReductionVals(VL);
return false;
}
- V.reorderTopToBottom();
+ // Do not reorder whole tree to preserve the original order of the ordered
+ // reduction.
V.reorderBottomToTop();
BoUpSLP::ExtraValueToDebugLocsMap LocalExternallyUsedValues;
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll b/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
index d807c68019877..d095ee31652f7 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-replaced.ll
@@ -5,13 +5,12 @@ define double @test(double %0) {
; CHECK-LABEL: define double @test(
; CHECK-SAME: double [[TMP0:%.*]]) {
; CHECK-NEXT: [[ENTRY:.*:]]
-; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> <double 1.000000e+00, double poison>, double [[TMP0]], i32 1
-; CHECK-NEXT: [[TMP2:%.*]] = fmul <2 x double> zeroinitializer, [[TMP1]]
-; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 1, i32 1>
-; CHECK-NEXT: [[TMP4:%.*]] = extractelement <4 x double> [[TMP3]], i32 1
-; CHECK-NEXT: [[TMP5:%.*]] = call reassoc double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[TMP3]])
+; CHECK-NEXT: [[TMP5:%.*]] = fmul double [[TMP0]], 0.000000e+00
+; CHECK-NEXT: [[TMP4:%.*]] = fadd reassoc double [[TMP5]], 0.000000e+00
; CHECK-NEXT: [[OP_RDX:%.*]] = fadd reassoc double [[TMP5]], [[TMP4]]
-; CHECK-NEXT: ret double [[OP_RDX]]
+; CHECK-NEXT: [[TMP6:%.*]] = fadd reassoc double [[TMP5]], [[OP_RDX]]
+; CHECK-NEXT: [[TMP7:%.*]] = fadd reassoc double [[TMP5]], [[TMP6]]
+; CHECK-NEXT: ret double [[TMP7]]
;
entry:
%1 = fmul double %0, 0.000000e+00
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-root-deleted.ll b/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-root-deleted.ll
index 3bffb0b8417f3..0b618f1d41d5c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-root-deleted.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/ordered-reduction-root-deleted.ll
@@ -27,7 +27,8 @@ define void @test(ptr %arg3, double %load.2.i, double %load.i, double %fadd10.i.
; CHECK-NEXT: [[TMP11:%.*]] = shufflevector <2 x double> [[TMP9]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
; CHECK-NEXT: [[TMP12:%.*]] = shufflevector <4 x double> [[TMP10]], <4 x double> [[TMP11]], <4 x i32> <i32 0, i32 4, i32 5, i32 3>
; CHECK-NEXT: [[TMP13:%.*]] = fmul <4 x double> [[TMP0]], [[TMP12]]
-; CHECK-NEXT: [[TMP14:%.*]] = call double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[TMP13]])
+; CHECK-NEXT: [[TMP18:%.*]] = shufflevector <4 x double> [[TMP13]], <4 x double> poison, <4 x i32> <i32 3, i32 0, i32 2, i32 1>
+; CHECK-NEXT: [[TMP14:%.*]] = call double @llvm.vector.reduce.fadd.v4f64(double -0.000000e+00, <4 x double> [[TMP18]])
; CHECK-NEXT: [[TMP15]] = fadd <2 x double> [[TMP7]], <double 0.000000e+00, double -0.000000e+00>
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <2 x double> [[TMP5]], double [[TMP14]], i32 1
; CHECK-NEXT: [[TMP17]] = fadd <2 x double> [[TMP16]], zeroinitializer
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
index 405efd15336d5..e12263b02103c 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reduction-ordered-fadd.ll
@@ -73,7 +73,8 @@ define float @reduce_ordered_fadd_reversed(ptr %a, ptr %b) {
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[A0]], align 4
; CHECK-NEXT: [[TMP1:%.*]] = load <8 x float>, ptr [[B0]], align 4
; CHECK-NEXT: [[TMP2:%.*]] = fmul <8 x float> [[TMP0]], [[TMP1]]
-; CHECK-NEXT: [[R6:%.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[TMP2]])
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <8 x float> [[TMP2]], <8 x float> poison, <8 x i32> <i32 7, i32 6, i32 5, i32 4, i32 3, i32 2, i32 1, i32 0>
+; CHECK-NEXT: [[R6:%.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[TMP3]])
; CHECK-NEXT: ret float [[R6]]
;
entry:
@@ -133,7 +134,7 @@ define float @reduce_ordered_fadd_reassoc_no_nsz(ptr %p) {
; CHECK-NEXT: [[ENTRY:.*:]]
; CHECK-NEXT: [[P0:%.*]] = getelementptr inbounds float, ptr [[P]], i64 0
; CHECK-NEXT: [[TMP0:%.*]] = load <8 x float>, ptr [[P0]], align 4
-; CHECK-NEXT: [[R6:%.*]] = call reassoc float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[TMP0]])
+; CHECK-NEXT: [[R6:%.*]] = call float @llvm.vector.reduce.fadd.v8f32(float -0.000000e+00, <8 x float> [[TMP0]])
; CHECK-NEXT: ret float [[R6]]
;
entry:
More information about the llvm-commits
mailing list