[llvm] [SLP][NFC]Add/update tests for fma-based transfromations, NFC (PR #225367)
Alexey Bataev via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 22 04:15:07 PDT 2026
https://github.com/alexey-bataev created https://github.com/llvm/llvm-project/pull/225367
None
>From 4874e0cc4a02b9c687940992cfa55546825d10d4 Mon Sep 17 00:00:00 2001
From: Alexey Bataev <a.bataev at outlook.com>
Date: Tue, 22 Sep 2026 04:14:51 -0700
Subject: [PATCH] =?UTF-8?q?[=F0=9D=98=80=F0=9D=97=BD=F0=9D=97=BF]=20initia?=
=?UTF-8?q?l=20version?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
Created using spr 1.3.7
---
.../AArch64/reassociate-fma-pairs.ll | 38 ++--
...tracts-folded-into-fmul-users-no-credit.ll | 152 +++++++++++++
.../fadd-with-gathered-fmul-operands.ll | 68 ++++++
.../fma-candidates-after-store-chains.ll | 96 ++++++++
.../fma-chain-no-alt-node-reduction.ll | 212 ++++++++++++++++++
.../fmul-constant-lane-fmuladd-combine.ll | 74 ++++++
.../AArch64/loop-accumulator-reduction.ll | 48 ++--
.../AArch64/reduction-root-multi-use-fma.ll | 58 +++++
.../X86/fmul-fused-into-scalar-fadd.ll | 89 ++++++++
.../X86/fsub-fmul-rhs-combine.ll | 77 +++++++
.../X86/horizontal-fadd-with-sub.ll | 17 +-
.../X86/reassoc-flattened-copyable-operand.ll | 109 ++++-----
.../X86/redux-feed-buildvector.ll | 70 +++---
.../X86/select-logical-or-and-i1-vector.ll | 42 ++--
14 files changed, 996 insertions(+), 154 deletions(-)
create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll
create mode 100644 llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll
diff --git a/llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll b/llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll
index 33df34fd80713..9631c6444cb2c 100644
--- a/llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll
+++ b/llvm/test/Transforms/PhaseOrdering/AArch64/reassociate-fma-pairs.ll
@@ -3,7 +3,11 @@
; This is derived from the calculations in a molecular dynamics benchmark. The
; reassociation of the final expression affects whether SLP can vectorize the
-; shared coefficient products and the energy update.
+; shared coefficient products and the energy update. With the FMA-aware SLP
+; cost model the shared products %ab/%bb and the last pair %p3/%q3 stay scalar:
+; vectorizing them would need lane extracts for the scalar users and would
+; break the fusion of %p3/%q3 into the fadd/fsub of the energy update
+; (fmadd/fmsub), so the loop body is emitted as scalar fmul/fmadd/fmsub.
define double @md_vdw_energy(ptr nocapture readonly %coeffs, double %energy, double %scale, double %table.delta, i64 %n) {
; CHECK-LABEL: define double @md_vdw_energy(
@@ -19,7 +23,7 @@ define double @md_vdw_energy(ptr nocapture readonly %coeffs, double %energy, dou
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[ACC1:%.*]] = phi double [ [[ENERGY]], %[[ENTRY]] ], [ [[RESULT1:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[ACC2:%.*]] = phi double [ [[ENERGY]], %[[ENTRY]] ], [ [[RESULT1:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[BASE:%.*]] = getelementptr inbounds [8 x i8], ptr [[COEFFS]], i64 [[I]]
; CHECK-NEXT: [[C0_PTR:%.*]] = getelementptr inbounds nuw i8, ptr [[BASE]], i64 16
; CHECK-NEXT: [[C0:%.*]] = load double, ptr [[C0_PTR]], align 8
@@ -37,28 +41,28 @@ define double @md_vdw_energy(ptr nocapture readonly %coeffs, double %energy, dou
; CHECK-NEXT: [[TMP3:%.*]] = load <2 x double>, ptr [[BASE]], align 8
; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <2 x double> [[TMP3]], [[TMP11]]
; CHECK-NEXT: [[TMP5:%.*]] = extractelement <2 x double> [[TMP4]], i64 0
-; CHECK-NEXT: [[P2:%.*]] = fmul fast double [[C0]], [[TMP5]]
+; CHECK-NEXT: [[ACC:%.*]] = fmul fast double [[C0]], [[TMP5]]
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x double> [[TMP4]], i64 1
-; CHECK-NEXT: [[Q2:%.*]] = fmul fast double [[C1]], [[TMP6]]
-; CHECK-NEXT: [[D2:%.*]] = fsub fast double [[P2]], [[Q2]]
+; CHECK-NEXT: [[TMP2:%.*]] = fmul fast double [[C1]], [[TMP6]]
+; CHECK-NEXT: [[D3_NEG:%.*]] = fsub fast double [[ACC]], [[TMP2]]
; CHECK-NEXT: [[P1:%.*]] = fmul fast double [[C2]], [[TMP5]]
; CHECK-NEXT: [[Q1:%.*]] = fmul fast double [[C3]], [[TMP6]]
; CHECK-NEXT: [[D1:%.*]] = fsub fast double [[P1]], [[Q1]]
-; CHECK-NEXT: [[ACC:%.*]] = fmul fast double [[C4]], [[TMP5]]
-; CHECK-NEXT: [[TMP2:%.*]] = fmul fast double [[C5]], [[TMP6]]
-; CHECK-NEXT: [[D3_NEG:%.*]] = fsub fast double [[ACC]], [[TMP2]]
+; CHECK-NEXT: [[P2:%.*]] = fmul fast double [[C4]], [[TMP5]]
+; CHECK-NEXT: [[Q2:%.*]] = fmul fast double [[C5]], [[TMP6]]
+; CHECK-NEXT: [[D2:%.*]] = fsub fast double [[P2]], [[Q2]]
; CHECK-NEXT: [[TMP7:%.*]] = load <2 x double>, ptr [[C6_PTR]], align 8
-; CHECK-NEXT: [[T1_REASS_REASS:%.*]] = fmul fast double [[FACTOR_OP_FMUL3]], [[D2]]
-; CHECK-NEXT: [[T2_REASS_REASS:%.*]] = fmul fast double [[D1]], [[FACTOR_OP_FMUL4]]
-; CHECK-NEXT: [[S1:%.*]] = fadd fast double [[T2_REASS_REASS]], [[T1_REASS_REASS]]
-; CHECK-NEXT: [[S2_NEG:%.*]] = fmul fast double [[D3_NEG]], [[FACTOR_OP_FMUL2]]
-; CHECK-NEXT: [[RESULT:%.*]] = fadd fast double [[S1]], [[S2_NEG]]
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x double> [[TMP7]], [[TMP4]]
-; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x double> [[TMP8]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = fmul fast double [[FACTOR_OP_FMUL3]], [[D3_NEG]]
+; CHECK-NEXT: [[RESULT:%.*]] = fmul fast double [[D1]], [[FACTOR_OP_FMUL4]]
; CHECK-NEXT: [[REASS_ADD:%.*]] = fadd fast double [[RESULT]], [[TMP9]]
-; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP8]], i64 1
+; CHECK-NEXT: [[T4_REASS:%.*]] = fmul fast double [[D2]], [[FACTOR_OP_FMUL2]]
+; CHECK-NEXT: [[ACC1:%.*]] = fadd fast double [[REASS_ADD]], [[T4_REASS]]
+; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <2 x double> [[TMP7]], [[TMP4]]
+; CHECK-NEXT: [[TMP10:%.*]] = extractelement <2 x double> [[TMP8]], i64 0
; CHECK-NEXT: [[S2_NEG1:%.*]] = fadd fast double [[ACC1]], [[TMP10]]
-; CHECK-NEXT: [[RESULT1]] = fsub fast double [[S2_NEG1]], [[REASS_ADD]]
+; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x double> [[TMP8]], i64 1
+; CHECK-NEXT: [[S2_NEG:%.*]] = fadd fast double [[ACC2]], [[TMP12]]
+; CHECK-NEXT: [[RESULT1]] = fsub fast double [[S2_NEG]], [[S2_NEG1]]
; CHECK-NEXT: [[NEXT]] = add nuw i64 [[I]], 10
; CHECK-NEXT: [[DONE_NOT:%.*]] = icmp ult i64 [[NEXT]], [[N]]
; CHECK-NEXT: br i1 [[DONE_NOT]], label %[[LOOP]], label %[[EXIT:.*]]
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll
new file mode 100644
index 0000000000000..f93a7dc3abf83
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/extracts-folded-into-fmul-users-no-credit.ll
@@ -0,0 +1,152 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -mcpu=neoverse-v2 < %s | FileCheck %s
+
+; The lane extracts feed scalar fmuls only, which the backend folds into
+; by-element fmuls (fmul d0, d1, v2.d[1]), so the extracts cost nothing in the
+; scalar code and their removal saves nothing. Without that credit the
+; 2-element part of the fmul reduction with the hoisted gathered operands is
+; not profitable: the scalar code is fmul + fmla + fmadd.
+define void @fmul_reduction_extracts_free_for_fmul_users(ptr %vp, ptr %sp, ptr %ap, ptr %bp, ptr %cp, ptr %dp, ptr %out, i64 %n) {
+; CHECK-LABEL: define void @fmul_reduction_extracts_free_for_fmul_users(
+; CHECK-SAME: ptr [[VP:%.*]], ptr [[SP:%.*]], ptr [[AP:%.*]], ptr [[BP:%.*]], ptr [[CP:%.*]], ptr [[DP:%.*]], ptr [[OUT:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[A:%.*]] = load double, ptr [[AP]], align 8
+; CHECK-NEXT: [[B:%.*]] = load double, ptr [[BP]], align 8
+; CHECK-NEXT: [[C:%.*]] = load double, ptr [[CP]], align 8
+; CHECK-NEXT: [[D:%.*]] = load double, ptr [[DP]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[A]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[B]], i64 1
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VPTR:%.*]] = getelementptr <2 x double>, ptr [[VP]], i64 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load <2 x double>, ptr [[VPTR]], align 16
+; CHECK-NEXT: [[SPTR:%.*]] = getelementptr double, ptr [[SP]], i64 [[I]]
+; CHECK-NEXT: [[S:%.*]] = load double, ptr [[SPTR]], align 8
+; CHECK-NEXT: [[E0:%.*]] = extractelement <2 x double> [[V]], i64 0
+; CHECK-NEXT: [[E1:%.*]] = extractelement <2 x double> [[V]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = fmul fast <2 x double> [[V]], [[TMP1]]
+; CHECK-NEXT: [[M2:%.*]] = fmul fast double [[S]], [[C]]
+; CHECK-NEXT: [[R0:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP2]])
+; CHECK-NEXT: [[R:%.*]] = fadd fast double [[R0]], [[M2]]
+; CHECK-NEXT: [[U0:%.*]] = fmul fast double [[E0]], [[S]]
+; CHECK-NEXT: [[U1:%.*]] = fmul fast double [[E1]], [[D]]
+; CHECK-NEXT: [[U2:%.*]] = fmul fast double [[E1]], [[S]]
+; CHECK-NEXT: [[O0:%.*]] = getelementptr double, ptr [[OUT]], i64 [[I]]
+; CHECK-NEXT: store double [[U0]], ptr [[O0]], align 8
+; CHECK-NEXT: [[I_N:%.*]] = add i64 [[I]], [[N]]
+; CHECK-NEXT: [[O1:%.*]] = getelementptr double, ptr [[OUT]], i64 [[I_N]]
+; CHECK-NEXT: store double [[U1]], ptr [[O1]], align 8
+; CHECK-NEXT: [[I_2N:%.*]] = add i64 [[I_N]], [[N]]
+; CHECK-NEXT: [[O2:%.*]] = getelementptr double, ptr [[OUT]], i64 [[I_2N]]
+; CHECK-NEXT: store double [[U2]], ptr [[O2]], align 8
+; CHECK-NEXT: [[I_NEXT]] = add nuw i64 [[I]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: [[LT:%.*]] = fcmp fast olt double [[R]], [[D]]
+; CHECK-NEXT: [[CONT:%.*]] = and i1 [[CMP]], [[LT]]
+; CHECK-NEXT: br i1 [[CONT]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %a = load double, ptr %ap, align 8
+ %b = load double, ptr %bp, align 8
+ %c = load double, ptr %cp, align 8
+ %d = load double, ptr %dp, align 8
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %vptr = getelementptr <2 x double>, ptr %vp, i64 %i
+ %v = load <2 x double>, ptr %vptr, align 16
+ %sptr = getelementptr double, ptr %sp, i64 %i
+ %s = load double, ptr %sptr, align 8
+ %e0 = extractelement <2 x double> %v, i64 0
+ %e1 = extractelement <2 x double> %v, i64 1
+ %m0 = fmul fast double %e0, %a
+ %m1 = fmul fast double %e1, %b
+ %m2 = fmul fast double %s, %c
+ %r0 = fadd fast double %m0, %m1
+ %r = fadd fast double %r0, %m2
+ %u0 = fmul fast double %e0, %s
+ %u1 = fmul fast double %e1, %d
+ %u2 = fmul fast double %e1, %s
+ %o0 = getelementptr double, ptr %out, i64 %i
+ store double %u0, ptr %o0, align 8
+ %i.n = add i64 %i, %n
+ %o1 = getelementptr double, ptr %out, i64 %i.n
+ store double %u1, ptr %o1, align 8
+ %i.2n = add i64 %i.n, %n
+ %o2 = getelementptr double, ptr %out, i64 %i.2n
+ store double %u2, ptr %o2, align 8
+ %i.next = add nuw i64 %i, 1
+ %cmp = icmp ult i64 %i.next, %n
+ %lt = fcmp fast olt double %r, %d
+ %cont = and i1 %cmp, %lt
+ br i1 %cont, label %loop, label %exit
+
+exit:
+ ret void
+}
+
+; The lane-1 extract feeding an fsub needs a real lane move, the extracts
+; disappear with the vectorization and their removal is credited: the
+; reduction is profitable.
+define void @fsub_reduction_extracts_credited(ptr %vp, ptr %sp, ptr %ap, ptr %bp, ptr %cp, ptr %dp, i64 %n) {
+; CHECK-LABEL: define void @fsub_reduction_extracts_credited(
+; CHECK-SAME: ptr [[VP:%.*]], ptr [[SP:%.*]], ptr [[AP:%.*]], ptr [[BP:%.*]], ptr [[CP:%.*]], ptr [[DP:%.*]], i64 [[N:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[A:%.*]] = load double, ptr [[AP]], align 8
+; CHECK-NEXT: [[B:%.*]] = load double, ptr [[BP]], align 8
+; CHECK-NEXT: [[C:%.*]] = load double, ptr [[CP]], align 8
+; CHECK-NEXT: [[D:%.*]] = load double, ptr [[DP]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x double> poison, double [[A]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP0]], double [[B]], i64 1
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[VPTR:%.*]] = getelementptr <2 x double>, ptr [[VP]], i64 [[I]]
+; CHECK-NEXT: [[V:%.*]] = load <2 x double>, ptr [[VPTR]], align 16
+; CHECK-NEXT: [[SPTR:%.*]] = getelementptr double, ptr [[SP]], i64 [[I]]
+; CHECK-NEXT: [[S:%.*]] = load double, ptr [[SPTR]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fsub fast <2 x double> [[V]], [[TMP1]]
+; CHECK-NEXT: [[M2:%.*]] = fmul fast double [[S]], [[C]]
+; CHECK-NEXT: [[TMP3:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP2]])
+; CHECK-NEXT: [[R:%.*]] = fadd fast double [[TMP3]], [[M2]]
+; CHECK-NEXT: [[I_NEXT]] = add nuw i64 [[I]], 1
+; CHECK-NEXT: [[CMP:%.*]] = icmp ult i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: [[LT:%.*]] = fcmp fast olt double [[R]], [[D]]
+; CHECK-NEXT: [[CONT:%.*]] = and i1 [[CMP]], [[LT]]
+; CHECK-NEXT: br i1 [[CONT]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %a = load double, ptr %ap, align 8
+ %b = load double, ptr %bp, align 8
+ %c = load double, ptr %cp, align 8
+ %d = load double, ptr %dp, align 8
+ br label %loop
+
+loop:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %loop ]
+ %vptr = getelementptr <2 x double>, ptr %vp, i64 %i
+ %v = load <2 x double>, ptr %vptr, align 16
+ %sptr = getelementptr double, ptr %sp, i64 %i
+ %s = load double, ptr %sptr, align 8
+ %e0 = extractelement <2 x double> %v, i64 0
+ %e1 = extractelement <2 x double> %v, i64 1
+ %m0 = fsub fast double %e0, %a
+ %m1 = fsub fast double %e1, %b
+ %m2 = fmul fast double %s, %c
+ %r0 = fadd fast double %m0, %m1
+ %r = fadd fast double %r0, %m2
+ %i.next = add nuw i64 %i, 1
+ %cmp = icmp ult i64 %i.next, %n
+ %lt = fcmp fast olt double %r, %d
+ %cont = and i1 %cmp, %lt
+ br i1 %cont, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll
new file mode 100644
index 0000000000000..86c59fa3ba652
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fadd-with-gathered-fmul-operands.ll
@@ -0,0 +1,68 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -mcpu=neoverse-v2 < %s | FileCheck %s
+
+; The fmul operands of the fadds are computed in different blocks, so they
+; cannot form a vector node and would be gathered. In the scalar code every
+; fadd fuses with its fmul into an fmadd, so a vectorized fadd lane only saves
+; (fmadd - fmul) = 0 on neoverse-v2: the vector fadd plus the gather of the
+; four products is not profitable and the fadds are not vectorized on their
+; own. Previously the scalar fadd lanes were priced as whole fmadds (2 each)
+; and the tree was vectorized.
+define void @fadd_with_gathered_fmul_operands(double %a0, double %b0, double %a1, double %b1, double %a2, double %b2, double %a3, double %b3, ptr %x, ptr %out) {
+;
+; CHECK-LABEL: define void @fadd_with_gathered_fmul_operands(
+; CHECK-SAME: double [[A0:%.*]], double [[B0:%.*]], double [[A1:%.*]], double [[B1:%.*]], double [[A2:%.*]], double [[B2:%.*]], double [[A3:%.*]], double [[B3:%.*]], ptr [[X:%.*]], ptr [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[M0:%.*]] = fmul contract double [[A0]], [[B0]]
+; CHECK-NEXT: br label %[[MUL1:.*]]
+; CHECK: [[MUL1]]:
+; CHECK-NEXT: [[M1:%.*]] = fmul contract double [[A1]], [[B1]]
+; CHECK-NEXT: br label %[[MUL2:.*]]
+; CHECK: [[MUL2]]:
+; CHECK-NEXT: [[M2:%.*]] = fmul contract double [[A2]], [[B2]]
+; CHECK-NEXT: br label %[[MUL3:.*]]
+; CHECK: [[MUL3]]:
+; CHECK-NEXT: [[M3:%.*]] = fmul contract double [[A3]], [[B3]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <4 x double>, ptr [[X]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <4 x double> poison, double [[M0]], i64 0
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> [[TMP1]], double [[M1]], i64 1
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[M2]], i64 2
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[M3]], i64 3
+; CHECK-NEXT: [[TMP5:%.*]] = fadd contract <4 x double> [[TMP4]], [[TMP0]]
+; CHECK-NEXT: store <4 x double> [[TMP5]], ptr [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %m0 = fmul contract double %a0, %b0
+ br label %mul1
+
+mul1:
+ %m1 = fmul contract double %a1, %b1
+ br label %mul2
+
+mul2:
+ %m2 = fmul contract double %a2, %b2
+ br label %mul3
+
+mul3:
+ %m3 = fmul contract double %a3, %b3
+ %x0 = load double, ptr %x, align 8
+ %x1p = getelementptr inbounds double, ptr %x, i64 1
+ %x1 = load double, ptr %x1p, align 8
+ %x2p = getelementptr inbounds double, ptr %x, i64 2
+ %x2 = load double, ptr %x2p, align 8
+ %x3p = getelementptr inbounds double, ptr %x, i64 3
+ %x3 = load double, ptr %x3p, align 8
+ %s0 = fadd contract double %m0, %x0
+ %s1 = fadd contract double %m1, %x1
+ %s2 = fadd contract double %m2, %x2
+ %s3 = fadd contract double %m3, %x3
+ store double %s0, ptr %out, align 8
+ %o1 = getelementptr inbounds double, ptr %out, i64 1
+ store double %s1, ptr %o1, align 8
+ %o2 = getelementptr inbounds double, ptr %out, i64 2
+ store double %s2, ptr %o2, align 8
+ %o3 = getelementptr inbounds double, ptr %out, i64 3
+ store double %s3, ptr %o3, align 8
+ ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll
new file mode 100644
index 0000000000000..9e63b6eb3dbce
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fma-candidates-after-store-chains.ll
@@ -0,0 +1,96 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -mcpu=neoverse-v2 < %s | FileCheck %s
+
+; %r is an FMA candidate: an fadd of two single-use contractable fmuls, which
+; the backend fuses into fmadd in the scalar code. The latch %acc.next PHI
+; reaches it through the incoming value %acc.body, and the latch is processed
+; (post order) before the body. The candidates are retried after all the
+; blocks of the function, not at the end of the latch, so the store chain of
+; the body is vectorized first: it takes the <%y, %z> pair (with the fsubs
+; and the loads) and the 2-lane reduction of %mx, %my reuses its vector.
+; Retried at the end of the latch, the reduction would take the <%x, %y> pair
+; first and the store chain would be left with a gather of an extract and a
+; scalar, and rejected.
+
+define double @test(ptr %a, ptr %h, ptr %out, ptr %q, i64 %n) {
+; CHECK-LABEL: define double @test(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[H:%.*]], ptr [[OUT:%.*]], ptr [[Q:%.*]], i64 [[N:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: br label %[[HEADER:.*]]
+; CHECK: [[HEADER]]:
+; CHECK-NEXT: [[I:%.*]] = phi i64 [ 0, %[[ENTRY]] ], [ [[I_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[ACC:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[ACC_NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[PC:%.*]] = getelementptr double, ptr [[Q]], i64 [[I]]
+; CHECK-NEXT: [[C:%.*]] = load double, ptr [[PC]], align 8
+; CHECK-NEXT: [[CMP:%.*]] = fcmp fast une double [[C]], 0.000000e+00
+; CHECK-NEXT: br i1 [[CMP]], label %[[BODY:.*]], label %[[LATCH]]
+; CHECK: [[BODY]]:
+; CHECK-NEXT: [[PA0:%.*]] = getelementptr double, ptr [[A]], i64 [[I]]
+; CHECK-NEXT: [[PA2:%.*]] = getelementptr double, ptr [[PA0]], i64 2
+; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[PA2]], align 8
+; CHECK-NEXT: [[X:%.*]] = fsub fast double [[A0]], 1.000000e+00
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[PA0]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = fsub fast <2 x double> [[TMP0]], splat (double 1.000000e+00)
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[H]], align 8
+; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP2]]
+; CHECK-NEXT: [[TMP6:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP5]])
+; CHECK-NEXT: [[ACC_BODY:%.*]] = fadd fast double [[TMP6]], [[ACC]]
+; CHECK-NEXT: [[TMP7:%.*]] = extractelement <2 x double> [[TMP1]], i64 1
+; CHECK-NEXT: [[FY:%.*]] = fmul fast double [[C]], [[TMP7]]
+; CHECK-NEXT: [[FZ:%.*]] = fmul fast double [[C]], [[X]]
+; CHECK-NEXT: [[POUT1:%.*]] = getelementptr double, ptr [[OUT]], i64 1
+; CHECK-NEXT: store double [[FY]], ptr [[OUT]], align 8
+; CHECK-NEXT: store double [[FZ]], ptr [[POUT1]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[ACC_NEXT]] = phi double [ [[ACC_BODY]], %[[BODY]] ], [ [[ACC]], %[[HEADER]] ]
+; CHECK-NEXT: [[I_NEXT]] = add nuw nsw i64 [[I]], 1
+; CHECK-NEXT: [[EXIT:%.*]] = icmp eq i64 [[I_NEXT]], [[N]]
+; CHECK-NEXT: br i1 [[EXIT]], label %[[END:.*]], label %[[HEADER]]
+; CHECK: [[END]]:
+; CHECK-NEXT: ret double [[ACC_NEXT]]
+;
+entry:
+ br label %header
+
+header:
+ %i = phi i64 [ 0, %entry ], [ %i.next, %latch ]
+ %acc = phi double [ 0.0, %entry ], [ %acc.next, %latch ]
+ %pc = getelementptr double, ptr %q, i64 %i
+ %c = load double, ptr %pc, align 8
+ %cmp = fcmp fast une double %c, 0.0
+ br i1 %cmp, label %body, label %latch
+
+body:
+ %pa0 = getelementptr double, ptr %a, i64 %i
+ %pa1 = getelementptr double, ptr %pa0, i64 1
+ %pa2 = getelementptr double, ptr %pa0, i64 2
+ %a0 = load double, ptr %pa0, align 8
+ %a1 = load double, ptr %pa1, align 8
+ %a2 = load double, ptr %pa2, align 8
+ %x = fsub fast double %a0, 1.0
+ %y = fsub fast double %a1, 1.0
+ %z = fsub fast double %a2, 1.0
+ %ph1 = getelementptr double, ptr %h, i64 1
+ %h0 = load double, ptr %h, align 8
+ %h1 = load double, ptr %ph1, align 8
+ %mx = fmul fast double %x, %h0
+ %my = fmul fast double %y, %h1
+ %r = fadd fast double %mx, %my
+ %acc.body = fadd fast double %r, %acc
+ %fy = fmul fast double %c, %y
+ %fz = fmul fast double %c, %z
+ %pout1 = getelementptr double, ptr %out, i64 1
+ store double %fy, ptr %out, align 8
+ store double %fz, ptr %pout1, align 8
+ br label %latch
+
+latch:
+ %acc.next = phi double [ %acc.body, %body ], [ %acc, %header ]
+ %i.next = add nuw nsw i64 %i, 1
+ %exit = icmp eq i64 %i.next, %n
+ br i1 %exit, label %end, label %header
+
+end:
+ ret double %acc.next
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll
new file mode 100644
index 0000000000000..66c89d60e23d9
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fma-chain-no-alt-node-reduction.ll
@@ -0,0 +1,212 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -passes=slp-vectorizer -S -mtriple=aarch64-unknown-linux-gnu -mcpu=neoverse-v2 < %s | FileCheck %s
+
+; Reduced from the density loop of Polyhedron mdbx MFORCE. For each neighbour
+; j of atom i the loop computes the periodic-wrapped coordinate differences
+; xij/yij/zij and then the quadratic form
+; rsq = xij*(g11*xij + g12d*yij + g13d*zij)
+; + yij*(g22*yij + g23d*zij)
+; + g33*zij*zij
+; where every fmul has a single contractable fadd user, i.e. the scalar code
+; is a chain of fmadd instructions on AArch64.
+;
+; SLP used to vectorize the rsq math as a shuffle-heavy alternate fmul/fadd
+; node rooted at the pair of fmuls (a2*xij, a3*yij) whose results feed the
+; scalar reduction fadds:
+; %v = fmul <2 x double> %a, %b
+; %w = fadd <2 x double> %a, %b
+; shufflevector %v, %w, <2 x i32> <i32 2, i32 1>
+; plus lane-blending shuffles for the operands and extractelements feeding the
+; remaining scalar fadds. This is slower than the scalar fmadd chain because
+; the fused multiply-adds are lost and replaced by shuffles.
+;
+; The cost model must account for the fmul+fadd -> fmadd fusion so that the
+; alternate node is not formed. The coordinate wrap chain (fsub/fcmp/select/
+; fsub/fcmp/fadd/select) for two of the three coordinates may still be
+; vectorized, and the rsq math must remain scalar (a plain 2-lane fmul feeding
+; a vector.reduce.fadd is acceptable).
+
+define void @density(ptr %lcs, ptr %liscom, ptr %densi, ptr %walls, i64 %first, i64 %cnt, double %xi, double %yi, double %zi, double %pbc, double %npbc, double %bx, double %by, double %bz, double %rcut) {
+; CHECK-LABEL: define void @density(
+; CHECK-SAME: ptr [[LCS:%.*]], ptr [[LISCOM:%.*]], ptr [[DENSI:%.*]], ptr [[WALLS:%.*]], i64 [[FIRST:%.*]], i64 [[CNT:%.*]], double [[XI:%.*]], double [[YI:%.*]], double [[ZI:%.*]], double [[PBC:%.*]], double [[NPBC:%.*]], double [[BX:%.*]], double [[BY:%.*]], double [[BZ:%.*]], double [[RCUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*]]:
+; CHECK-NEXT: [[P72:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 72
+; CHECK-NEXT: [[G11:%.*]] = load double, ptr [[P72]], align 8
+; CHECK-NEXT: [[P104:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 104
+; CHECK-NEXT: [[P136:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 136
+; CHECK-NEXT: [[G33:%.*]] = load double, ptr [[P136]], align 8
+; CHECK-NEXT: [[P96:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 96
+; CHECK-NEXT: [[W96:%.*]] = load double, ptr [[P96]], align 8
+; CHECK-NEXT: [[P80:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 80
+; CHECK-NEXT: [[W80:%.*]] = load double, ptr [[P80]], align 8
+; CHECK-NEXT: [[G12D:%.*]] = fadd fast double [[W80]], [[W96]]
+; CHECK-NEXT: [[P120:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 120
+; CHECK-NEXT: [[P88:%.*]] = getelementptr inbounds nuw i8, ptr [[WALLS]], i64 88
+; CHECK-NEXT: [[W88:%.*]] = load double, ptr [[P88]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[P120]], align 8
+; CHECK-NEXT: [[TMP8:%.*]] = load <2 x double>, ptr [[P104]], align 8
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <2 x double> [[TMP8]], double [[W88]], i64 0
+; CHECK-NEXT: [[TMP19:%.*]] = fadd fast <2 x double> [[TMP18]], [[TMP0]]
+; CHECK-NEXT: [[TMP4:%.*]] = shufflevector <2 x double> [[TMP8]], <2 x double> poison, <2 x i32> <i32 poison, i32 0>
+; CHECK-NEXT: [[TMP21:%.*]] = insertelement <2 x double> poison, double [[XI]], i64 0
+; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x double> [[TMP21]], double [[YI]], i64 1
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[PBC]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = shufflevector <2 x double> [[TMP2]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP22:%.*]] = insertelement <2 x double> poison, double [[BX]], i64 0
+; CHECK-NEXT: [[TMP5:%.*]] = insertelement <2 x double> [[TMP22]], double [[BY]], i64 1
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <2 x double> poison, double [[NPBC]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x double> [[TMP6]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP23:%.*]] = insertelement <2 x double> poison, double [[G11]], i64 0
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <2 x double> [[TMP23]], double [[G12D]], i64 1
+; CHECK-NEXT: br label %[[LOOP:.*]]
+; CHECK: [[LOOP]]:
+; CHECK-NEXT: [[IV:%.*]] = phi i64 [ [[FIRST]], %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LATCH:.*]] ]
+; CHECK-NEXT: [[C:%.*]] = phi i64 [ [[CNT]], %[[ENTRY]] ], [ [[C_NEXT:%.*]], %[[LATCH]] ]
+; CHECK-NEXT: [[GEP_J:%.*]] = getelementptr [4 x i8], ptr [[LISCOM]], i64 [[IV]]
+; CHECK-NEXT: [[GEP_J1:%.*]] = getelementptr i8, ptr [[GEP_J]], i64 -4
+; CHECK-NEXT: [[J32:%.*]] = load i32, ptr [[GEP_J1]], align 4
+; CHECK-NEXT: [[J:%.*]] = sext i32 [[J32]] to i64
+; CHECK-NEXT: [[J_IDX:%.*]] = mul nsw i64 [[J]], 24
+; CHECK-NEXT: [[LCS_J:%.*]] = getelementptr i8, ptr [[LCS]], i64 [[J_IDX]]
+; CHECK-NEXT: [[PX:%.*]] = getelementptr i8, ptr [[LCS_J]], i64 48
+; CHECK-NEXT: [[PZ:%.*]] = getelementptr i8, ptr [[LCS_J]], i64 64
+; CHECK-NEXT: [[ZJ:%.*]] = load double, ptr [[PZ]], align 8
+; CHECK-NEXT: [[DX:%.*]] = fsub fast double [[ZI]], [[ZJ]]
+; CHECK-NEXT: [[CX:%.*]] = fcmp fast ogt double [[DX]], [[PBC]]
+; CHECK-NEXT: [[SX:%.*]] = select ninf nsz i1 [[CX]], double [[BZ]], double 0.000000e+00
+; CHECK-NEXT: [[DX1:%.*]] = fsub reassoc nsz arcp contract afn double [[DX]], [[SX]]
+; CHECK-NEXT: [[CX2:%.*]] = fcmp fast olt double [[DX1]], [[NPBC]]
+; CHECK-NEXT: [[DX2:%.*]] = fadd fast double [[DX1]], [[BZ]]
+; CHECK-NEXT: [[XIJ:%.*]] = select nsz i1 [[CX2]], double [[DX2]], double [[DX1]]
+; CHECK-NEXT: [[TMP10:%.*]] = load <2 x double>, ptr [[PX]], align 8
+; CHECK-NEXT: [[TMP11:%.*]] = fsub fast <2 x double> [[TMP1]], [[TMP10]]
+; CHECK-NEXT: [[TMP12:%.*]] = fcmp fast ogt <2 x double> [[TMP11]], [[TMP3]]
+; CHECK-NEXT: [[TMP13:%.*]] = select <2 x i1> [[TMP12]], <2 x double> [[TMP5]], <2 x double> zeroinitializer
+; CHECK-NEXT: [[TMP14:%.*]] = fsub reassoc nsz arcp contract afn <2 x double> [[TMP11]], [[TMP13]]
+; CHECK-NEXT: [[TMP15:%.*]] = fcmp fast olt <2 x double> [[TMP14]], [[TMP7]]
+; CHECK-NEXT: [[TMP16:%.*]] = fadd fast <2 x double> [[TMP14]], [[TMP5]]
+; CHECK-NEXT: [[TMP17:%.*]] = select <2 x i1> [[TMP15]], <2 x double> [[TMP16]], <2 x double> [[TMP14]]
+; CHECK-NEXT: [[TMP20:%.*]] = fmul fast <2 x double> [[TMP17]], [[TMP9]]
+; CHECK-NEXT: [[TMP24:%.*]] = shufflevector <2 x double> [[TMP17]], <2 x double> [[TMP20]], <2 x i32> <i32 3, i32 1>
+; CHECK-NEXT: [[TMP25:%.*]] = shufflevector <2 x double> [[TMP4]], <2 x double> [[TMP20]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT: [[TMP26:%.*]] = fmul fast <2 x double> [[TMP24]], [[TMP25]]
+; CHECK-NEXT: [[TMP27:%.*]] = fadd fast <2 x double> [[TMP24]], [[TMP25]]
+; CHECK-NEXT: [[TMP28:%.*]] = shufflevector <2 x double> [[TMP26]], <2 x double> [[TMP27]], <2 x i32> <i32 2, i32 1>
+; CHECK-NEXT: [[TMP29:%.*]] = insertelement <2 x double> poison, double [[XIJ]], i64 0
+; CHECK-NEXT: [[TMP30:%.*]] = shufflevector <2 x double> [[TMP29]], <2 x double> poison, <2 x i32> zeroinitializer
+; CHECK-NEXT: [[TMP31:%.*]] = fmul fast <2 x double> [[TMP30]], [[TMP19]]
+; CHECK-NEXT: [[TMP32:%.*]] = fadd fast <2 x double> [[TMP31]], [[TMP28]]
+; CHECK-NEXT: [[TMP33:%.*]] = fmul fast <2 x double> [[TMP32]], [[TMP17]]
+; CHECK-NEXT: [[ZZ:%.*]] = fmul fast double [[XIJ]], [[XIJ]]
+; CHECK-NEXT: [[TZ:%.*]] = fmul fast double [[ZZ]], [[G33]]
+; CHECK-NEXT: [[TY:%.*]] = extractelement <2 x double> [[TMP33]], i64 1
+; CHECK-NEXT: [[A4:%.*]] = fadd fast double [[TY]], [[TZ]]
+; CHECK-NEXT: [[TX:%.*]] = extractelement <2 x double> [[TMP33]], i64 0
+; CHECK-NEXT: [[RSQ:%.*]] = fadd fast double [[A4]], [[TX]]
+; CHECK-NEXT: [[CMP:%.*]] = fcmp fast olt double [[RSQ]], [[RCUT]]
+; CHECK-NEXT: br i1 [[CMP]], label %[[ACCUM:.*]], label %[[LATCH]]
+; CHECK: [[ACCUM]]:
+; CHECK-NEXT: [[DI:%.*]] = load double, ptr [[DENSI]], align 8
+; CHECK-NEXT: [[DI1:%.*]] = fadd fast double [[DI]], [[RSQ]]
+; CHECK-NEXT: store double [[DI1]], ptr [[DENSI]], align 8
+; CHECK-NEXT: br label %[[LATCH]]
+; CHECK: [[LATCH]]:
+; CHECK-NEXT: [[IV_NEXT]] = add nsw i64 [[IV]], 1
+; CHECK-NEXT: [[C_NEXT]] = add nsw i64 [[C]], -1
+; CHECK-NEXT: [[CONT:%.*]] = icmp sgt i64 [[C]], 1
+; CHECK-NEXT: br i1 [[CONT]], label %[[LOOP]], label %[[EXIT:.*]]
+; CHECK: [[EXIT]]:
+; CHECK-NEXT: ret void
+;
+entry:
+ %p72 = getelementptr inbounds nuw i8, ptr %walls, i64 72
+ %g11 = load double, ptr %p72, align 8
+ %p104 = getelementptr inbounds nuw i8, ptr %walls, i64 104
+ %g22 = load double, ptr %p104, align 8
+ %p136 = getelementptr inbounds nuw i8, ptr %walls, i64 136
+ %g33 = load double, ptr %p136, align 8
+ %p96 = getelementptr inbounds nuw i8, ptr %walls, i64 96
+ %w96 = load double, ptr %p96, align 8
+ %p80 = getelementptr inbounds nuw i8, ptr %walls, i64 80
+ %w80 = load double, ptr %p80, align 8
+ %g12d = fadd fast double %w80, %w96
+ %p120 = getelementptr inbounds nuw i8, ptr %walls, i64 120
+ %w120 = load double, ptr %p120, align 8
+ %p88 = getelementptr inbounds nuw i8, ptr %walls, i64 88
+ %w88 = load double, ptr %p88, align 8
+ %g13d = fadd fast double %w88, %w120
+ %p128 = getelementptr inbounds nuw i8, ptr %walls, i64 128
+ %w128 = load double, ptr %p128, align 8
+ %p112 = getelementptr inbounds nuw i8, ptr %walls, i64 112
+ %w112 = load double, ptr %p112, align 8
+ %g23d = fadd fast double %w112, %w128
+ br label %loop
+
+loop:
+ %iv = phi i64 [ %first, %entry ], [ %iv.next, %latch ]
+ %c = phi i64 [ %cnt, %entry ], [ %c.next, %latch ]
+ %gep.j = getelementptr [4 x i8], ptr %liscom, i64 %iv
+ %gep.j1 = getelementptr i8, ptr %gep.j, i64 -4
+ %j32 = load i32, ptr %gep.j1, align 4
+ %j = sext i32 %j32 to i64
+ %j.idx = mul nsw i64 %j, 24
+ %lcs.j = getelementptr i8, ptr %lcs, i64 %j.idx
+ %px = getelementptr i8, ptr %lcs.j, i64 48
+ %xj = load double, ptr %px, align 8
+ %dx = fsub fast double %xi, %xj
+ %cx = fcmp fast ogt double %dx, %pbc
+ %sx = select ninf nsz i1 %cx, double %bx, double 0.000000e+00
+ %dx1 = fsub reassoc nsz arcp contract afn double %dx, %sx
+ %cx2 = fcmp fast olt double %dx1, %npbc
+ %dx2 = fadd fast double %dx1, %bx
+ %xij = select nsz i1 %cx2, double %dx2, double %dx1
+ %py = getelementptr i8, ptr %lcs.j, i64 56
+ %yj = load double, ptr %py, align 8
+ %dy = fsub fast double %yi, %yj
+ %cy = fcmp fast ogt double %dy, %pbc
+ %sy = select ninf nsz i1 %cy, double %by, double 0.000000e+00
+ %dy1 = fsub reassoc nsz arcp contract afn double %dy, %sy
+ %cy2 = fcmp fast olt double %dy1, %npbc
+ %dy2 = fadd fast double %dy1, %by
+ %yij = select nsz i1 %cy2, double %dy2, double %dy1
+ %pz = getelementptr i8, ptr %lcs.j, i64 64
+ %zj = load double, ptr %pz, align 8
+ %dz = fsub fast double %zi, %zj
+ %cz = fcmp fast ogt double %dz, %pbc
+ %sz = select ninf nsz i1 %cz, double %bz, double 0.000000e+00
+ %dz1 = fsub reassoc nsz arcp contract afn double %dz, %sz
+ %cz2 = fcmp fast olt double %dz1, %npbc
+ %dz2 = fadd fast double %dz1, %bz
+ %zij = select nsz i1 %cz2, double %dz2, double %dz1
+ %m1 = fmul fast double %xij, %g11
+ %m2 = fmul fast double %yij, %g12d
+ %a1 = fadd fast double %m2, %m1
+ %m3 = fmul fast double %zij, %g13d
+ %a2 = fadd fast double %a1, %m3
+ %tx = fmul fast double %a2, %xij
+ %m4 = fmul fast double %yij, %g22
+ %m5 = fmul fast double %zij, %g23d
+ %a3 = fadd fast double %m5, %m4
+ %ty = fmul fast double %a3, %yij
+ %zz = fmul fast double %zij, %zij
+ %tz = fmul fast double %zz, %g33
+ %a4 = fadd fast double %ty, %tz
+ %rsq = fadd fast double %a4, %tx
+ %cmp = fcmp fast olt double %rsq, %rcut
+ br i1 %cmp, label %accum, label %latch
+
+accum:
+ %di = load double, ptr %densi, align 8
+ %di1 = fadd fast double %di, %rsq
+ store double %di1, ptr %densi, align 8
+ br label %latch
+
+latch:
+ %iv.next = add nsw i64 %iv, 1
+ %c.next = add nsw i64 %c, -1
+ %cont = icmp sgt i64 %c, 1
+ br i1 %cont, label %loop, label %exit
+
+exit:
+ ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll
new file mode 100644
index 0000000000000..95ae52224e29b
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/fmul-constant-lane-fmuladd-combine.ll
@@ -0,0 +1,74 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -mcpu=neoverse-v2 < %s | FileCheck %s
+
+; Derived from loop-accumulator-reduction.ll (loop_acc_fadd) without the
+; loop-carried accumulator: three rows l0*c_k + l1*c_k+1 + l2*c_k+2 are stored
+; together with a constant 0.0 to four consecutive locations. In the 4-wide
+; tree the fmul nodes have three l*c lanes and a constant lane (the copyable
+; lane of the constant store, -0.0 * 1.0) and feed the fadd nodes. Such fmul
+; nodes now form the combined fmuladd node with their fadd users, so the
+; scalar rows are priced as fmadd chains and the tree, whose coefficients are
+; gathered from the arguments, is not profitable. Previously the constant lane
+; blocked the combine, the scalar rows were priced as separate fmul and fadd
+; and the tree was vectorized 4-wide.
+define void @fmul_constant_lane_rows(ptr %p, ptr %out, double %c0, double %c1, double %c2, double %c3, double %c4, double %c5, double %c6, double %c7, double %c8) {
+; CHECK-LABEL: define void @fmul_constant_lane_rows(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[OUT:%.*]], double [[C0:%.*]], double [[C1:%.*]], double [[C2:%.*]], double [[C3:%.*]], double [[C4:%.*]], double [[C5:%.*]], double [[C6:%.*]], double [[C7:%.*]], double [[C8:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[P2:%.*]] = getelementptr double, ptr [[P]], i64 2
+; CHECK-NEXT: [[L2:%.*]] = load double, ptr [[P2]], align 8
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> <double undef, double -0.000000e+00>, <4 x i32> <i32 0, i32 0, i32 0, i32 3>
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C0]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[C3]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[C6]], i64 2
+; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <4 x double> [[TMP1]], [[TMP4]]
+; CHECK-NEXT: [[TMP6:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <4 x i32> <i32 poison, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <2 x double> [[TMP0]], <2 x double> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> <double poison, double 0.000000e+00, double poison, double poison>, <4 x double> [[TMP7]], <4 x i32> <i32 5, i32 1, i32 poison, i32 poison>
+; CHECK-NEXT: [[TMP9:%.*]] = shufflevector <4 x double> [[TMP8]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C1]], i64 0
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP10]], double [[C4]], i64 1
+; CHECK-NEXT: [[TMP12:%.*]] = insertelement <4 x double> [[TMP11]], double [[C7]], i64 2
+; CHECK-NEXT: [[TMP13:%.*]] = fmul fast <4 x double> [[TMP9]], [[TMP12]]
+; CHECK-NEXT: [[TMP14:%.*]] = fadd fast <4 x double> [[TMP13]], [[TMP5]]
+; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L2]], i64 0
+; CHECK-NEXT: [[TMP16:%.*]] = shufflevector <4 x double> [[TMP15]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C2]], i64 0
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[C5]], i64 1
+; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x double> [[TMP18]], double [[C8]], i64 2
+; CHECK-NEXT: [[TMP20:%.*]] = fmul fast <4 x double> [[TMP16]], [[TMP19]]
+; CHECK-NEXT: [[TMP21:%.*]] = fadd fast <4 x double> [[TMP14]], [[TMP20]]
+; CHECK-NEXT: store <4 x double> [[TMP21]], ptr [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %l0 = load double, ptr %p, align 8
+ %p1 = getelementptr double, ptr %p, i64 1
+ %l1 = load double, ptr %p1, align 8
+ %p2 = getelementptr double, ptr %p, i64 2
+ %l2 = load double, ptr %p2, align 8
+ %a00 = fmul fast double %l0, %c0
+ %a01 = fmul fast double %l1, %c1
+ %a02 = fadd fast double %a01, %a00
+ %a03 = fmul fast double %l2, %c2
+ %a = fadd fast double %a02, %a03
+ %b00 = fmul fast double %l0, %c3
+ %b01 = fmul fast double %l1, %c4
+ %b02 = fadd fast double %b01, %b00
+ %b03 = fmul fast double %l2, %c5
+ %b = fadd fast double %b02, %b03
+ %d00 = fmul fast double %l0, %c6
+ %d01 = fmul fast double %l1, %c7
+ %d02 = fadd fast double %d01, %d00
+ %d03 = fmul fast double %l2, %c8
+ %d = fadd fast double %d02, %d03
+ store double %a, ptr %out, align 8
+ %o1 = getelementptr double, ptr %out, i64 1
+ store double %b, ptr %o1, align 8
+ %o2 = getelementptr double, ptr %out, i64 2
+ store double %d, ptr %o2, align 8
+ %o3 = getelementptr double, ptr %out, i64 3
+ store double 0.000000e+00, ptr %o3, align 8
+ ret void
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
index af3baf9d29fbc..f03bd6227982b 100644
--- a/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/loop-accumulator-reduction.ll
@@ -422,25 +422,25 @@ define double @root_stored(ptr %p, i64 %n, ptr %o) {
; CHECK-NEXT: [[TMP3:%.*]] = insertelement <4 x double> [[TMP2]], double [[C3]], i64 1
; CHECK-NEXT: [[TMP4:%.*]] = insertelement <4 x double> [[TMP3]], double [[C6]], i64 2
; CHECK-NEXT: [[TMP5:%.*]] = fmul fast <4 x double> [[TMP1]], [[TMP4]]
-; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> poison, double [[L1]], i64 0
-; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> [[TMP11]], double [[ACC]], i64 1
-; CHECK-NEXT: [[TMP7:%.*]] = shufflevector <4 x double> [[TMP6]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C1]], i64 0
-; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> [[TMP8]], double [[C4]], i64 1
-; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> [[TMP9]], double [[C7]], i64 2
-; CHECK-NEXT: [[TMP12:%.*]] = fmul reassoc nsz arcp contract afn <4 x double> [[TMP7]], [[TMP10]]
-; CHECK-NEXT: [[TMP24:%.*]] = fadd reassoc nsz arcp contract afn <4 x double> [[TMP12]], [[TMP5]]
-; CHECK-NEXT: [[TMP13:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L2]], i64 0
-; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x double> [[TMP13]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
-; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C2]], i64 0
-; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> [[TMP15]], double [[C5]], i64 1
-; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[C8]], i64 2
-; CHECK-NEXT: [[TMP18:%.*]] = fmul fast <4 x double> [[TMP14]], [[TMP17]]
-; CHECK-NEXT: [[TMP19:%.*]] = fadd reassoc nsz arcp contract afn <4 x double> [[TMP24]], [[TMP18]]
-; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[TMP19]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 poison, i32 7>
-; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP20]], <4 x double> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
-; CHECK-NEXT: [[TMP22:%.*]] = fmul <4 x double> [[TMP19]], [[TMP21]]
-; CHECK-NEXT: [[TMP23]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP22]])
+; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x double> poison, double [[L1]], i64 0
+; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x double> [[TMP6]], double [[ACC]], i64 1
+; CHECK-NEXT: [[TMP8:%.*]] = shufflevector <4 x double> [[TMP7]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
+; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C1]], i64 0
+; CHECK-NEXT: [[TMP10:%.*]] = insertelement <4 x double> [[TMP9]], double [[C4]], i64 1
+; CHECK-NEXT: [[TMP11:%.*]] = insertelement <4 x double> [[TMP10]], double [[C7]], i64 2
+; CHECK-NEXT: [[TMP12:%.*]] = fmul reassoc nsz arcp contract afn <4 x double> [[TMP8]], [[TMP11]]
+; CHECK-NEXT: [[TMP13:%.*]] = fadd reassoc nsz arcp contract afn <4 x double> [[TMP12]], [[TMP5]]
+; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x double> <double poison, double -0.000000e+00, double poison, double poison>, double [[L2]], i64 0
+; CHECK-NEXT: [[TMP15:%.*]] = shufflevector <4 x double> [[TMP14]], <4 x double> poison, <4 x i32> <i32 0, i32 0, i32 0, i32 1>
+; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, double [[C2]], i64 0
+; CHECK-NEXT: [[TMP17:%.*]] = insertelement <4 x double> [[TMP16]], double [[C5]], i64 1
+; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x double> [[TMP17]], double [[C8]], i64 2
+; CHECK-NEXT: [[TMP19:%.*]] = fmul fast <4 x double> [[TMP15]], [[TMP18]]
+; CHECK-NEXT: [[TMP20:%.*]] = fadd reassoc nsz arcp contract afn <4 x double> [[TMP13]], [[TMP19]]
+; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP20]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 poison, i32 7>
+; CHECK-NEXT: [[TMP22:%.*]] = shufflevector <4 x double> [[TMP21]], <4 x double> [[TMP20]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
+; CHECK-NEXT: [[TMP24:%.*]] = fmul <4 x double> [[TMP20]], [[TMP22]]
+; CHECK-NEXT: [[TMP23]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP24]])
; CHECK-NEXT: store double [[TMP23]], ptr [[O]], align 8
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], [[N]]
@@ -511,7 +511,7 @@ define double @two_exit_phis(ptr %p, i64 %n, i1 %c) {
; CHECK-NEXT: br label %[[LOOP:.*]]
; CHECK: [[LOOP]]:
; CHECK-NEXT: [[IV:%.*]] = phi i64 [ 1, %[[ENTRY]] ], [ [[IV_NEXT:%.*]], %[[LOOP]] ]
-; CHECK-NEXT: [[ACC:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[OP_RDX:%.*]], %[[LOOP]] ]
+; CHECK-NEXT: [[SD:%.*]] = phi double [ 0.000000e+00, %[[ENTRY]] ], [ [[SUM:%.*]], %[[LOOP]] ]
; CHECK-NEXT: [[IV3:%.*]] = mul nuw i64 [[IV]], 3
; CHECK-NEXT: [[P0:%.*]] = getelementptr double, ptr [[P]], i64 [[IV3]]
; CHECK-NEXT: [[P1:%.*]] = getelementptr double, ptr [[P0]], i64 1
@@ -551,14 +551,14 @@ define double @two_exit_phis(ptr %p, i64 %n, i1 %c) {
; CHECK-NEXT: [[TMP20:%.*]] = shufflevector <4 x double> [[TMP19]], <4 x double> <double poison, double poison, double poison, double 1.000000e+00>, <4 x i32> <i32 0, i32 1, i32 poison, i32 7>
; CHECK-NEXT: [[TMP21:%.*]] = shufflevector <4 x double> [[TMP20]], <4 x double> [[TMP19]], <4 x i32> <i32 0, i32 1, i32 6, i32 3>
; CHECK-NEXT: [[TMP22:%.*]] = fmul <4 x double> [[TMP19]], [[TMP21]]
-; CHECK-NEXT: [[TMP25:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP22]])
-; CHECK-NEXT: [[OP_RDX]] = fadd fast double [[TMP25]], [[ACC]]
+; CHECK-NEXT: [[T2:%.*]] = call fast double @llvm.vector.reduce.fadd.v4f64(double 0.000000e+00, <4 x double> [[TMP22]])
+; CHECK-NEXT: [[SUM]] = fadd fast double [[T2]], [[SD]]
; CHECK-NEXT: [[IV_NEXT]] = add nuw nsw i64 [[IV]], 1
; CHECK-NEXT: [[CMP:%.*]] = icmp eq i64 [[IV]], [[N]]
; CHECK-NEXT: br i1 [[CMP]], label %[[EXIT:.*]], label %[[LOOP]]
; CHECK: [[EXIT]]:
-; CHECK-NEXT: [[TMP23:%.*]] = phi double [ [[OP_RDX]], %[[LOOP]] ]
-; CHECK-NEXT: [[TMP24:%.*]] = phi double [ [[OP_RDX]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP23:%.*]] = phi double [ [[SUM]], %[[LOOP]] ]
+; CHECK-NEXT: [[TMP24:%.*]] = phi double [ [[SUM]], %[[LOOP]] ]
; CHECK-NEXT: [[R:%.*]] = fadd double [[TMP23]], [[TMP24]]
; CHECK-NEXT: ret double [[R]]
;
diff --git a/llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll b/llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll
new file mode 100644
index 0000000000000..0b998f1efd678
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/AArch64/reduction-root-multi-use-fma.ll
@@ -0,0 +1,58 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 6
+; RUN: opt -S -passes=slp-vectorizer -mtriple=aarch64-unknown-linux-gnu -mcpu=neoverse-v2 < %s | FileCheck %s
+
+; The root of the reduction has several users, the backend still fuses its
+; fmul operand into it (fmul + fmadd in the scalar code). Priced as a plain
+; fadd, the scalar code looked more expensive than the vector fmul + faddp
+; with the gathered operands.
+define double @fmul_reduction_root_multi_use(ptr %p, double %a, double %b) {
+; CHECK-LABEL: define double @fmul_reduction_root_multi_use(
+; CHECK-SAME: ptr [[P:%.*]], double [[A:%.*]], double [[B:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[A]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[B]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[R:%.*]] = call fast double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP4]])
+; CHECK-NEXT: [[U0:%.*]] = fmul fast double [[R]], [[A]]
+; CHECK-NEXT: [[U1:%.*]] = fmul fast double [[R]], [[B]]
+; CHECK-NEXT: [[RES:%.*]] = fdiv fast double [[U0]], [[U1]]
+; CHECK-NEXT: ret double [[RES]]
+;
+ %p1 = getelementptr inbounds double, ptr %p, i64 1
+ %l0 = load double, ptr %p, align 8
+ %l1 = load double, ptr %p1, align 8
+ %m0 = fmul fast double %l0, %a
+ %m1 = fmul fast double %l1, %b
+ %r = fadd fast double %m0, %m1
+ %u0 = fmul fast double %r, %a
+ %u1 = fmul fast double %r, %b
+ %res = fdiv fast double %u0, %u1
+ ret double %res
+}
+
+; The fadd does not contract, the fmuls stay separate in the scalar code and
+; the vector code is profitable.
+define double @fmul_reduction_root_multi_use_no_contract(ptr %p, double %a, double %b) {
+; CHECK-LABEL: define double @fmul_reduction_root_multi_use_no_contract(
+; CHECK-SAME: ptr [[P:%.*]], double [[A:%.*]], double [[B:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = insertelement <2 x double> poison, double [[A]], i64 0
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> [[TMP2]], double [[B]], i64 1
+; CHECK-NEXT: [[TMP4:%.*]] = fmul reassoc nsz <2 x double> [[TMP1]], [[TMP3]]
+; CHECK-NEXT: [[TMP5:%.*]] = call reassoc nsz double @llvm.vector.reduce.fadd.v2f64(double 0.000000e+00, <2 x double> [[TMP4]])
+; CHECK-NEXT: [[U0:%.*]] = fmul reassoc nsz double [[TMP5]], [[A]]
+; CHECK-NEXT: [[U1:%.*]] = fmul reassoc nsz double [[TMP5]], [[B]]
+; CHECK-NEXT: [[RES:%.*]] = fdiv reassoc nsz double [[U0]], [[U1]]
+; CHECK-NEXT: ret double [[RES]]
+;
+ %p1 = getelementptr inbounds double, ptr %p, i64 1
+ %l0 = load double, ptr %p, align 8
+ %l1 = load double, ptr %p1, align 8
+ %m0 = fmul reassoc nsz double %l0, %a
+ %m1 = fmul reassoc nsz double %l1, %b
+ %r = fadd reassoc nsz double %m0, %m1
+ %u0 = fmul reassoc nsz double %r, %a
+ %u1 = fmul reassoc nsz double %r, %b
+ %res = fdiv reassoc nsz double %u0, %u1
+ ret double %res
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll b/llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll
new file mode 100644
index 0000000000000..9609a9a370b46
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fmul-fused-into-scalar-fadd.ll
@@ -0,0 +1,89 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v3 < %s | FileCheck %s
+
+; Two independent products of vector-loadable inputs whose only users are
+; contractable scalar fadds, returned on different paths. The backend fuses
+; each product into its fadd (fmadd), so the products have to stay scalar: a
+; vector fmul with two extracts breaks both fusions.
+
+; The fmul is the first operand of the fadd: the fusion was already recognized
+; and the products are not vectorized (decision preserved).
+define double @fmul_lhs_of_scalar_fadd(ptr %p, ptr %q, double %x, double %y, i1 %c) {
+; CHECK-LABEL: define double @fmul_lhs_of_scalar_fadd(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], double [[X:%.*]], double [[Y:%.*]], i1 [[C:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[A0:%.*]] = load double, ptr [[P]], align 8
+; CHECK-NEXT: [[P1:%.*]] = getelementptr inbounds double, ptr [[P]], i64 1
+; CHECK-NEXT: [[A1:%.*]] = load double, ptr [[P1]], align 8
+; CHECK-NEXT: [[B0:%.*]] = load double, ptr [[Q]], align 8
+; CHECK-NEXT: [[Q1:%.*]] = getelementptr inbounds double, ptr [[Q]], i64 1
+; CHECK-NEXT: [[B1:%.*]] = load double, ptr [[Q1]], align 8
+; CHECK-NEXT: [[M0:%.*]] = fmul contract double [[A0]], [[B0]]
+; CHECK-NEXT: [[M1:%.*]] = fmul contract double [[A1]], [[B1]]
+; CHECK-NEXT: br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
+; CHECK: [[T]]:
+; CHECK-NEXT: [[S0:%.*]] = fadd contract double [[M0]], [[X]]
+; CHECK-NEXT: ret double [[S0]]
+; CHECK: [[F]]:
+; CHECK-NEXT: [[S1:%.*]] = fadd contract double [[M1]], [[Y]]
+; CHECK-NEXT: ret double [[S1]]
+;
+entry:
+ %a0 = load double, ptr %p, align 8
+ %p1 = getelementptr inbounds double, ptr %p, i64 1
+ %a1 = load double, ptr %p1, align 8
+ %b0 = load double, ptr %q, align 8
+ %q1 = getelementptr inbounds double, ptr %q, i64 1
+ %b1 = load double, ptr %q1, align 8
+ %m0 = fmul contract double %a0, %b0
+ %m1 = fmul contract double %a1, %b1
+ br i1 %c, label %t, label %f
+
+t:
+ %s0 = fadd contract double %m0, %x
+ ret double %s0
+
+f:
+ %s1 = fadd contract double %m1, %y
+ ret double %s1
+}
+
+; The fmul is the second operand of the fadd: it used to be vectorized with
+; two extracts, the fusion was only recognized for the first operand. It is
+; fused into the scalar fadd just the same and stays scalar now.
+define double @fmul_rhs_of_scalar_fadd(ptr %p, ptr %q, double %x, double %y, i1 %c) {
+; CHECK-LABEL: define double @fmul_rhs_of_scalar_fadd(
+; CHECK-SAME: ptr [[P:%.*]], ptr [[Q:%.*]], double [[X:%.*]], double [[Y:%.*]], i1 [[C:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[P]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Q]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: br i1 [[C]], label %[[T:.*]], label %[[F:.*]]
+; CHECK: [[T]]:
+; CHECK-NEXT: [[M0:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
+; CHECK-NEXT: [[S0:%.*]] = fadd contract double [[X]], [[M0]]
+; CHECK-NEXT: ret double [[S0]]
+; CHECK: [[F]]:
+; CHECK-NEXT: [[M1:%.*]] = extractelement <2 x double> [[TMP2]], i64 1
+; CHECK-NEXT: [[S1:%.*]] = fadd contract double [[Y]], [[M1]]
+; CHECK-NEXT: ret double [[S1]]
+;
+entry:
+ %a0 = load double, ptr %p, align 8
+ %p1 = getelementptr inbounds double, ptr %p, i64 1
+ %a1 = load double, ptr %p1, align 8
+ %b0 = load double, ptr %q, align 8
+ %q1 = getelementptr inbounds double, ptr %q, i64 1
+ %b1 = load double, ptr %q1, align 8
+ %m0 = fmul contract double %a0, %b0
+ %m1 = fmul contract double %a1, %b1
+ br i1 %c, label %t, label %f
+
+t:
+ %s0 = fadd contract double %x, %m0
+ ret double %s0
+
+f:
+ %s1 = fadd contract double %y, %m1
+ ret double %s1
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll b/llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll
new file mode 100644
index 0000000000000..b766d92bd13fb
--- /dev/null
+++ b/llvm/test/Transforms/SLPVectorizer/X86/fsub-fmul-rhs-combine.ll
@@ -0,0 +1,77 @@
+; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
+; RUN: opt -S -passes=slp-vectorizer -mtriple=x86_64-unknown-linux-gnu -mcpu=x86-64-v3 < %s | FileCheck %s
+
+; c - a*b with the fmul in the subtrahend. The backend fuses it into an fmsub
+; just like a*b - c, so the fmul/fsub pair forms the combined fmuladd node for
+; the second operand too.
+
+; All operands are loaded and the results are stored: the whole tree is
+; vectorized as before (decision preserved). The combined node is emitted as a
+; plain fsub of the vector fmul.
+define void @fsub_fmul_rhs_all_loads(ptr %a, ptr %b, ptr %c, ptr %out) {
+; CHECK-LABEL: define void @fsub_fmul_rhs_all_loads(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], ptr [[C:%.*]], ptr [[OUT:%.*]]) #[[ATTR0:[0-9]+]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[B]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = load <2 x double>, ptr [[C]], align 8
+; CHECK-NEXT: [[TMP3:%.*]] = fmul contract <2 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP4:%.*]] = fsub contract <2 x double> [[TMP2]], [[TMP3]]
+; CHECK-NEXT: store <2 x double> [[TMP4]], ptr [[OUT]], align 8
+; CHECK-NEXT: ret void
+;
+entry:
+ %a0 = load double, ptr %a, align 8
+ %a1p = getelementptr inbounds double, ptr %a, i64 1
+ %a1 = load double, ptr %a1p, align 8
+ %b0 = load double, ptr %b, align 8
+ %b1p = getelementptr inbounds double, ptr %b, i64 1
+ %b1 = load double, ptr %b1p, align 8
+ %c0 = load double, ptr %c, align 8
+ %c1p = getelementptr inbounds double, ptr %c, i64 1
+ %c1 = load double, ptr %c1p, align 8
+ %m0 = fmul contract double %a0, %b0
+ %m1 = fmul contract double %a1, %b1
+ %s0 = fsub contract double %c0, %m0
+ %s1 = fsub contract double %c1, %m1
+ store double %s0, ptr %out, align 8
+ %o1 = getelementptr inbounds double, ptr %out, i64 1
+ store double %s1, ptr %o1, align 8
+ ret void
+}
+
+; Borderline: c is gathered from arguments and the results feed a scalar user
+; through two extracts. With the combined node the fmul node is absorbed into
+; the fsub node and the scalar fmul/fsub pairs count as fused fmadds, so the
+; 2-wide tree needs more vector instructions than the scalar code and is
+; rejected. Previously the fmul and fsub nodes were priced separately and the
+; tree was vectorized.
+define double @fsub_fmul_rhs_gathered_c(ptr %a, ptr %b, double %c0, double %c1) {
+; CHECK-LABEL: define double @fsub_fmul_rhs_gathered_c(
+; CHECK-SAME: ptr [[A:%.*]], ptr [[B:%.*]], double [[C0:%.*]], double [[C1:%.*]]) #[[ATTR0]] {
+; CHECK-NEXT: [[ENTRY:.*:]]
+; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[A]], align 8
+; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[B]], align 8
+; CHECK-NEXT: [[TMP2:%.*]] = fmul contract <2 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP3:%.*]] = insertelement <2 x double> poison, double [[C0]], i64 0
+; CHECK-NEXT: [[TMP4:%.*]] = insertelement <2 x double> [[TMP3]], double [[C1]], i64 1
+; CHECK-NEXT: [[TMP5:%.*]] = fsub contract <2 x double> [[TMP4]], [[TMP2]]
+; CHECK-NEXT: [[S0:%.*]] = extractelement <2 x double> [[TMP5]], i64 0
+; CHECK-NEXT: [[S1:%.*]] = extractelement <2 x double> [[TMP5]], i64 1
+; CHECK-NEXT: [[R:%.*]] = fmul double [[S0]], [[S1]]
+; CHECK-NEXT: ret double [[R]]
+;
+entry:
+ %a0 = load double, ptr %a, align 8
+ %a1p = getelementptr inbounds double, ptr %a, i64 1
+ %a1 = load double, ptr %a1p, align 8
+ %b0 = load double, ptr %b, align 8
+ %b1p = getelementptr inbounds double, ptr %b, i64 1
+ %b1 = load double, ptr %b1p, align 8
+ %m0 = fmul contract double %a0, %b0
+ %m1 = fmul contract double %a1, %b1
+ %s0 = fsub contract double %c0, %m0
+ %s1 = fsub contract double %c1, %m1
+ %r = fmul double %s0, %s1
+ ret double %r
+}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
index e8c566ecdaabc..0a452665d7480 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/horizontal-fadd-with-sub.ll
@@ -573,6 +573,9 @@ entry:
; too few unordered candidates, the analysis switches to the ordered
; reduction, discarding the tracked signs - it must restart without the
; fsub flattening (no vector fsub of the z values is emitted).
+; The multiplies are not contractable: otherwise the scalar fmul/fsub and
+; fmul/fadd pairs fuse into fma and vectorizing the multiplies is not
+; profitable, which would hide the restart.
define double @ordered_switch_restart(ptr %x, ptr %y, ptr %z, ptr %out) {
; CHECK-LABEL: define double @ordered_switch_restart(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]], ptr [[OUT:%.*]]) #[[ATTR0]] {
@@ -581,7 +584,7 @@ define double @ordered_switch_restart(ptr %x, ptr %y, ptr %z, ptr %out) {
; CHECK-NEXT: [[Z0:%.*]] = load double, ptr [[Z]], align 8
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP1]], [[TMP0]]
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz <2 x double> [[TMP1]], [[TMP0]]
; CHECK-NEXT: [[Z1:%.*]] = load double, ptr [[Z8]], align 8
; CHECK-NEXT: [[ZSUM:%.*]] = fadd reassoc nsz contract double [[Z0]], [[Z1]]
; CHECK-NEXT: store double [[ZSUM]], ptr [[OUT]], align 8
@@ -597,11 +600,11 @@ entry:
%z8 = getelementptr inbounds nuw i8, ptr %z, i64 8
%x0 = load double, ptr %x, align 8
%y0 = load double, ptr %y, align 8
- %mul = fmul reassoc nsz contract double %y0, %x0
+ %mul = fmul reassoc nsz double %y0, %x0
%z0 = load double, ptr %z, align 8
%x1 = load double, ptr %x8, align 8
%y1 = load double, ptr %y8, align 8
- %mul5 = fmul reassoc nsz contract double %y1, %x1
+ %mul5 = fmul reassoc nsz double %y1, %x1
%z1 = load double, ptr %z8, align 8
%zsum = fadd reassoc nsz contract double %z0, %z1
store double %zsum, ptr %out, align 8
@@ -1266,6 +1269,8 @@ entry:
; The scaled repeated value (m0 * 2.0) must not carry contract: as an fma
; candidate, the following fadd would block the pairwise vectorization of the
; multiplies.
+; The multiplies themselves are not contractable: otherwise m1 fuses into an
+; fma with its fadd user and vectorizing the pair is not profitable.
define double @scaled_value_no_contract(ptr %x, ptr %y, ptr %z, ptr %w) {
; CHECK-LABEL: define double @scaled_value_no_contract(
; CHECK-SAME: ptr [[X:%.*]], ptr [[Y:%.*]], ptr [[Z:%.*]], ptr [[W:%.*]]) #[[ATTR0]] {
@@ -1273,7 +1278,7 @@ define double @scaled_value_no_contract(ptr %x, ptr %y, ptr %z, ptr %w) {
; CHECK-NEXT: [[Z0:%.*]] = load double, ptr [[Z]], align 8
; CHECK-NEXT: [[TMP0:%.*]] = load <2 x double>, ptr [[X]], align 8
; CHECK-NEXT: [[TMP1:%.*]] = load <2 x double>, ptr [[Y]], align 8
-; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz contract <2 x double> [[TMP0]], [[TMP1]]
+; CHECK-NEXT: [[TMP2:%.*]] = fmul reassoc nsz <2 x double> [[TMP0]], [[TMP1]]
; CHECK-NEXT: [[W0:%.*]] = load double, ptr [[W]], align 8
; CHECK-NEXT: [[N:%.*]] = fneg reassoc nsz contract double [[W0]]
; CHECK-NEXT: [[TMP3:%.*]] = extractelement <2 x double> [[TMP2]], i64 0
@@ -1289,11 +1294,11 @@ entry:
%y1 = getelementptr inbounds double, ptr %y, i64 1
%x0v = load double, ptr %x, align 8
%y0v = load double, ptr %y, align 8
- %m0 = fmul reassoc nsz contract double %x0v, %y0v
+ %m0 = fmul reassoc nsz double %x0v, %y0v
%z0 = load double, ptr %z, align 8
%x1v = load double, ptr %x1, align 8
%y1v = load double, ptr %y1, align 8
- %m1 = fmul reassoc nsz contract double %x1v, %y1v
+ %m1 = fmul reassoc nsz double %x1v, %y1v
%w0 = load double, ptr %w, align 8
%n = fneg reassoc nsz contract double %w0
%s0 = fadd reassoc nsz contract double %m1, %n
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/reassoc-flattened-copyable-operand.ll b/llvm/test/Transforms/SLPVectorizer/X86/reassoc-flattened-copyable-operand.ll
index 585bfa7717094..c1067696ecf72 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/reassoc-flattened-copyable-operand.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/reassoc-flattened-copyable-operand.ll
@@ -1,5 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5
; RUN: opt -S --passes=slp-vectorizer < %s -mtriple=x86_64-unknown-linux-gnu -mcpu=znver4 | FileCheck %s
+; The fmul/fadd/fsub ops are deliberately not contractable (fast minus contract): with
+; contract the fadd/fsub lanes fuse with their single-use fmul operands into scalar FMAs
+; and the cost model no longer forms the flattened fadd/fsub node this test exercises.
@hexsh_ = external global [4496 x i8]
@@ -8,26 +11,26 @@ define void @test(ptr %0, ptr %1, ptr %2, ptr %3, ptr %4, ptr %5, ptr %6, ptr %7
; CHECK-SAME: ptr [[TMP0:%.*]], ptr [[TMP1:%.*]], ptr [[TMP2:%.*]], ptr [[TMP3:%.*]], ptr [[TMP4:%.*]], ptr [[TMP5:%.*]], ptr [[TMP6:%.*]], ptr [[TMP7:%.*]], ptr [[_QMSHARED_COMMON_DATAEP2TH:%.*]], double [[TMP8:%.*]], double [[TMP9:%.*]], double [[TMP10:%.*]], double [[TMP11:%.*]], double [[TMP12:%.*]], ptr [[_QMSHARED_COMMON_DATAEPONE:%.*]], double [[TMP13:%.*]], double [[TMP14:%.*]], double [[TMP15:%.*]], ptr [[TMP16:%.*]], double [[TMP17:%.*]], double [[TMP18:%.*]], double [[TMP19:%.*]], double [[TMP20:%.*]], double [[TMP21:%.*]], double [[TMP22:%.*]], double [[TMP23:%.*]], double [[TMP24:%.*]], double [[TMP25:%.*]], ptr [[_QMSHARED_COMMON_DATAEPTWO:%.*]], ptr [[_QMSHARED_COMMON_DATAEPTHR:%.*]], ptr [[_QMSHARED_COMMON_DATAEP72TH:%.*]], ptr [[TMP26:%.*]], double [[TMP27:%.*]], double [[TMP28:%.*]], double [[TMP29:%.*]], double [[TMP30:%.*]], double [[TMP31:%.*]], double [[TMP32:%.*]], double [[TMP33:%.*]], double [[TMP34:%.*]], double [[TMP35:%.*]], double [[TMP36:%.*]], double [[TMP37:%.*]], double [[TMP38:%.*]], double [[TMP39:%.*]], double [[TMP40:%.*]], double [[TMP41:%.*]], double [[TMP42:%.*]], double [[TMP43:%.*]], double [[TMP44:%.*]], double [[TMP45:%.*]], double [[TMP46:%.*]], double [[TMP47:%.*]], double [[TMP48:%.*]]) #[[ATTR0:[0-9]+]] {
; CHECK-NEXT: [[_PREHEADER4179_PREHEADER:.*:]]
; CHECK-NEXT: store double 1.000000e+00, ptr [[TMP26]], align 8
-; CHECK-NEXT: [[TMP49:%.*]] = fadd fast double [[TMP38]], 1.000000e+00
-; CHECK-NEXT: [[TMP50:%.*]] = fadd fast double [[TMP42]], 1.000000e+00
+; CHECK-NEXT: [[TMP49:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP38]], 1.000000e+00
+; CHECK-NEXT: [[TMP50:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP42]], 1.000000e+00
; CHECK-NEXT: store double [[TMP44]], ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 224), align 8
-; CHECK-NEXT: [[TMP51:%.*]] = fmul fast double [[TMP8]], [[TMP10]]
-; CHECK-NEXT: [[TMP52:%.*]] = fmul fast double [[TMP8]], [[TMP9]]
-; CHECK-NEXT: [[TMP53:%.*]] = fneg fast double [[TMP25]]
-; CHECK-NEXT: [[TMP62:%.*]] = fmul fast double [[TMP13]], [[TMP51]]
-; CHECK-NEXT: [[TMP54:%.*]] = fadd fast double [[TMP51]], 1.000000e+00
-; CHECK-NEXT: [[TMP61:%.*]] = fmul fast double [[TMP13]], [[TMP52]]
-; CHECK-NEXT: [[TMP55:%.*]] = fmul fast double [[TMP48]], [[TMP40]]
-; CHECK-NEXT: [[TMP56:%.*]] = fmul fast double [[TMP36]], [[TMP46]]
-; CHECK-NEXT: [[TMP57:%.*]] = fmul fast double [[TMP32]], [[TMP40]]
-; CHECK-NEXT: [[TMP58:%.*]] = fmul fast double [[TMP39]], [[TMP40]]
-; CHECK-NEXT: [[TMP59:%.*]] = fadd fast double [[TMP56]], 1.000000e+00
-; CHECK-NEXT: [[TMP63:%.*]] = fmul fast double [[TMP36]], [[TMP62]]
-; CHECK-NEXT: [[TMP64:%.*]] = fmul fast double [[TMP36]], [[TMP61]]
-; CHECK-NEXT: [[TMP60:%.*]] = fmul fast double [[TMP24]], [[TMP53]]
-; CHECK-NEXT: [[TMP65:%.*]] = fadd fast double [[TMP63]], 2.000000e+00
-; CHECK-NEXT: [[TMP66:%.*]] = fmul fast double [[TMP45]], [[TMP61]]
-; CHECK-NEXT: [[TMP69:%.*]] = fsub fast double [[TMP64]], [[TMP59]]
+; CHECK-NEXT: [[TMP51:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP8]], [[TMP10]]
+; CHECK-NEXT: [[TMP52:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP8]], [[TMP9]]
+; CHECK-NEXT: [[TMP53:%.*]] = fneg reassoc nnan ninf nsz arcp afn double [[TMP25]]
+; CHECK-NEXT: [[TMP62:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP13]], [[TMP51]]
+; CHECK-NEXT: [[TMP54:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP51]], 1.000000e+00
+; CHECK-NEXT: [[TMP61:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP13]], [[TMP52]]
+; CHECK-NEXT: [[TMP55:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP48]], [[TMP40]]
+; CHECK-NEXT: [[TMP58:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP36]], [[TMP46]]
+; CHECK-NEXT: [[TMP59:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP32]], [[TMP40]]
+; CHECK-NEXT: [[TMP64:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP39]], [[TMP40]]
+; CHECK-NEXT: [[TMP67:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP58]], 1.000000e+00
+; CHECK-NEXT: [[TMP68:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP36]], [[TMP62]]
+; CHECK-NEXT: [[TMP63:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP36]], [[TMP61]]
+; CHECK-NEXT: [[TMP60:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP24]], [[TMP53]]
+; CHECK-NEXT: [[TMP65:%.*]] = fadd reassoc nnan ninf nsz arcp afn double [[TMP68]], 2.000000e+00
+; CHECK-NEXT: [[TMP66:%.*]] = fmul reassoc nnan ninf nsz arcp afn double [[TMP45]], [[TMP61]]
+; CHECK-NEXT: [[TMP69:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[TMP63]], [[TMP67]]
; CHECK-NEXT: [[TMP70:%.*]] = insertelement <4 x double> poison, double [[TMP69]], i64 0
; CHECK-NEXT: [[TMP71:%.*]] = insertelement <4 x double> [[TMP70]], double [[TMP66]], i64 1
; CHECK-NEXT: [[TMP72:%.*]] = insertelement <4 x double> [[TMP71]], double [[TMP65]], i64 2
@@ -36,11 +39,11 @@ define void @test(ptr %0, ptr %1, ptr %2, ptr %3, ptr %4, ptr %5, ptr %6, ptr %7
; CHECK-NEXT: [[TMP75:%.*]] = insertelement <4 x double> [[TMP74]], double [[TMP55]], i64 1
; CHECK-NEXT: [[TMP76:%.*]] = insertelement <4 x double> [[TMP75]], double [[TMP61]], i64 2
; CHECK-NEXT: [[TMP77:%.*]] = insertelement <4 x double> [[TMP76]], double [[TMP54]], i64 3
-; CHECK-NEXT: [[TMP78:%.*]] = fadd fast <4 x double> [[TMP73]], [[TMP77]]
-; CHECK-NEXT: [[TMP79:%.*]] = fsub fast <4 x double> [[TMP73]], [[TMP77]]
+; CHECK-NEXT: [[TMP78:%.*]] = fadd reassoc nnan ninf nsz arcp afn <4 x double> [[TMP73]], [[TMP77]]
+; CHECK-NEXT: [[TMP79:%.*]] = fsub reassoc nnan ninf nsz arcp afn <4 x double> [[TMP73]], [[TMP77]]
; CHECK-NEXT: [[TMP80:%.*]] = shufflevector <4 x double> [[TMP78]], <4 x double> [[TMP79]], <4 x i32> <i32 0, i32 5, i32 6, i32 7>
-; CHECK-NEXT: [[TMP81:%.*]] = fsub fast double [[TMP50]], [[TMP57]]
-; CHECK-NEXT: [[TMP82:%.*]] = fsub fast double [[TMP49]], [[TMP58]]
+; CHECK-NEXT: [[TMP81:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[TMP50]], [[TMP59]]
+; CHECK-NEXT: [[TMP82:%.*]] = fsub reassoc nnan ninf nsz arcp afn double [[TMP49]], [[TMP64]]
; CHECK-NEXT: [[TMP83:%.*]] = insertelement <8 x double> poison, double [[TMP35]], i64 0
; CHECK-NEXT: [[TMP84:%.*]] = insertelement <8 x double> [[TMP83]], double [[TMP82]], i64 1
; CHECK-NEXT: [[TMP85:%.*]] = insertelement <8 x double> [[TMP84]], double [[TMP81]], i64 2
@@ -49,49 +52,49 @@ define void @test(ptr %0, ptr %1, ptr %2, ptr %3, ptr %4, ptr %5, ptr %6, ptr %7
; CHECK-NEXT: [[TMP88:%.*]] = shufflevector <8 x double> [[TMP86]], <8 x double> [[TMP87]], <8 x i32> <i32 0, i32 1, i32 2, i32 3, i32 8, i32 9, i32 10, i32 11>
; CHECK-NEXT: [[TMP89:%.*]] = insertelement <8 x double> <double 1.000000e+00, double poison, double 1.000000e+00, double poison, double 1.000000e+00, double 1.000000e+00, double poison, double poison>, double [[TMP27]], i64 1
; CHECK-NEXT: [[TMP90:%.*]] = shufflevector <8 x double> [[TMP89]], <8 x double> poison, <8 x i32> <i32 0, i32 1, i32 2, i32 1, i32 4, i32 5, i32 1, i32 1>
-; CHECK-NEXT: [[TMP91:%.*]] = fmul reassoc nsz arcp contract afn <8 x double> [[TMP88]], [[TMP90]]
+; CHECK-NEXT: [[TMP91:%.*]] = fmul reassoc nsz arcp afn <8 x double> [[TMP88]], [[TMP90]]
; CHECK-NEXT: store <8 x double> [[TMP91]], ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 136), align 8
; CHECK-NEXT: ret void
;
.preheader4179.preheader:
- %49 = fmul fast double %8, %9
- %50 = fmul fast double %8, %10
- %51 = fmul fast double %13, %49
- %52 = fmul fast double %13, %50
- %53 = fneg fast double %25
- %54 = fmul fast double %24, %53
+ %49 = fmul reassoc nnan ninf nsz arcp afn double %8, %9
+ %50 = fmul reassoc nnan ninf nsz arcp afn double %8, %10
+ %51 = fmul reassoc nnan ninf nsz arcp afn double %13, %49
+ %52 = fmul reassoc nnan ninf nsz arcp afn double %13, %50
+ %53 = fneg reassoc nnan ninf nsz arcp afn double %25
+ %54 = fmul reassoc nnan ninf nsz arcp afn double %24, %53
store double 1.000000e+00, ptr %26, align 8
- %55 = fadd fast double %50, 1.000000e+00
- %56 = fsub fast double %54, %55
- %57 = fmul fast double %56, %27
+ %55 = fadd reassoc nnan ninf nsz arcp afn double %50, 1.000000e+00
+ %56 = fsub reassoc nnan ninf nsz arcp afn double %54, %55
+ %57 = fmul reassoc nnan ninf nsz arcp afn double %56, %27
store double %57, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 192), align 8
store double %35, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 136), align 8
- %58 = fadd fast double %38, 1.000000e+00
- %59 = fmul fast double %39, %40
- %60 = fsub fast double %58, %59
- %61 = fmul fast double %60, %27
+ %58 = fadd reassoc nnan ninf nsz arcp afn double %38, 1.000000e+00
+ %59 = fmul reassoc nnan ninf nsz arcp afn double %39, %40
+ %60 = fsub reassoc nnan ninf nsz arcp afn double %58, %59
+ %61 = fmul reassoc nnan ninf nsz arcp afn double %60, %27
store double %61, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 144), align 8
- %62 = fadd fast double %42, 1.000000e+00
- %63 = fmul fast double %32, %40
- %64 = fsub fast double %62, %63
+ %62 = fadd reassoc nnan ninf nsz arcp afn double %42, 1.000000e+00
+ %63 = fmul reassoc nnan ninf nsz arcp afn double %32, %40
+ %64 = fsub reassoc nnan ninf nsz arcp afn double %62, %63
store double %64, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 152), align 8
store double %44, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 224), align 8
- %65 = fmul fast double %45, %51
- %66 = fmul fast double %52, %27
+ %65 = fmul reassoc nnan ninf nsz arcp afn double %45, %51
+ %66 = fmul reassoc nnan ninf nsz arcp afn double %52, %27
store double %66, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 160), align 8
- %67 = fmul fast double %36, %46
- %68 = fmul fast double %36, %51
- %69 = fadd fast double %67, 1.000000e+00
- %70 = fsub fast double %68, %69
- %71 = fadd fast double %70, %47
+ %67 = fmul reassoc nnan ninf nsz arcp afn double %36, %46
+ %68 = fmul reassoc nnan ninf nsz arcp afn double %36, %51
+ %69 = fadd reassoc nnan ninf nsz arcp afn double %67, 1.000000e+00
+ %70 = fsub reassoc nnan ninf nsz arcp afn double %68, %69
+ %71 = fadd reassoc nnan ninf nsz arcp afn double %70, %47
store double %71, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 168), align 8
- %72 = fmul fast double %48, %40
- %73 = fsub fast double %65, %72
+ %72 = fmul reassoc nnan ninf nsz arcp afn double %48, %40
+ %73 = fsub reassoc nnan ninf nsz arcp afn double %65, %72
store double %73, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 176), align 8
- %74 = fmul fast double %36, %52
- %75 = fadd fast double %74, 2.000000e+00
- %76 = fsub fast double %75, %51
- %77 = fmul fast double %76, %27
+ %74 = fmul reassoc nnan ninf nsz arcp afn double %36, %52
+ %75 = fadd reassoc nnan ninf nsz arcp afn double %74, 2.000000e+00
+ %76 = fsub reassoc nnan ninf nsz arcp afn double %75, %51
+ %77 = fmul reassoc nnan ninf nsz arcp afn double %76, %27
store double %77, ptr getelementptr inbounds nuw (i8, ptr @hexsh_, i64 184), align 8
ret void
}
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll b/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
index 67054d202d5a2..aba0af9218327 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/redux-feed-buildvector.ll
@@ -4,6 +4,8 @@
; The test represents the case with multiple vectorization possibilities
; but the most effective way to vectorize it is to match both 8-way reductions
; feeding the insertelement vector build sequence.
+; The fmul/fadd ops deliberately lack the contract flag so they cannot fuse
+; into fmadd, keeping the scalar and vector costs comparable.
declare void @llvm.masked.scatter.v2f64.v2p0(<2 x double>, <2 x ptr>, i32 immarg, <2 x i1>)
@@ -16,11 +18,11 @@ define void @test(ptr nocapture readonly %arg, ptr nocapture readonly %arg1, ptr
; CHECK-NEXT: [[GEP2_0:%.*]] = getelementptr inbounds double, ptr [[ARG1:%.*]], i64 16
; CHECK-NEXT: [[TMP2:%.*]] = call <8 x double> @llvm.masked.gather.v8f64.v8p0(<8 x ptr> align 8 [[TMP5]], <8 x i1> splat (i1 true), <8 x double> poison)
; CHECK-NEXT: [[TMP3:%.*]] = load <8 x double>, ptr [[GEP2_0]], align 8
-; CHECK-NEXT: [[TMP4:%.*]] = fmul fast <8 x double> [[TMP3]], [[TMP2]]
+; CHECK-NEXT: [[TMP8:%.*]] = fmul reassoc nnan ninf nsz arcp afn <8 x double> [[TMP3]], [[TMP2]]
; CHECK-NEXT: [[TMP6:%.*]] = load <8 x double>, ptr [[ARG1]], align 8
-; CHECK-NEXT: [[TMP8:%.*]] = fmul fast <8 x double> [[TMP6]], [[TMP2]]
-; CHECK-NEXT: [[TMP7:%.*]] = call fast double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP8]])
-; CHECK-NEXT: [[TMP11:%.*]] = call fast double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP4]])
+; CHECK-NEXT: [[TMP9:%.*]] = fmul reassoc nnan ninf nsz arcp afn <8 x double> [[TMP6]], [[TMP2]]
+; CHECK-NEXT: [[TMP7:%.*]] = call reassoc nnan ninf nsz arcp afn double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP9]])
+; CHECK-NEXT: [[TMP11:%.*]] = call reassoc nnan ninf nsz arcp afn double @llvm.vector.reduce.fadd.v8f64(double 0.000000e+00, <8 x double> [[TMP8]])
; CHECK-NEXT: [[I142:%.*]] = insertelement <2 x double> poison, double [[TMP7]], i64 0
; CHECK-NEXT: [[I143:%.*]] = insertelement <2 x double> [[I142]], double [[TMP11]], i64 1
; CHECK-NEXT: [[P:%.*]] = getelementptr inbounds double, ptr [[ARG2:%.*]], <2 x i64> <i64 0, i64 16>
@@ -31,80 +33,80 @@ entry:
%gep1.0 = getelementptr inbounds double, ptr %arg, i64 1
%ld1.0 = load double, ptr %gep1.0, align 8
%ld0.0 = load double, ptr %arg1, align 8
- %mul1.0 = fmul fast double %ld0.0, %ld1.0
+ %mul1.0 = fmul reassoc nnan ninf nsz arcp afn double %ld0.0, %ld1.0
%gep2.0 = getelementptr inbounds double, ptr %arg1, i64 16
%ld2.0 = load double, ptr %gep2.0, align 8
- %mul2.0 = fmul fast double %ld2.0, %ld1.0
+ %mul2.0 = fmul reassoc nnan ninf nsz arcp afn double %ld2.0, %ld1.0
%gep1.1 = getelementptr inbounds double, ptr %arg, i64 3
%ld1.1 = load double, ptr %gep1.1, align 8
%gep0.1 = getelementptr inbounds double, ptr %arg1, i64 1
%ld0.1 = load double, ptr %gep0.1, align 8
- %mul1.1 = fmul fast double %ld0.1, %ld1.1
- %rdx1.0 = fadd fast double %mul1.0, %mul1.1
+ %mul1.1 = fmul reassoc nnan ninf nsz arcp afn double %ld0.1, %ld1.1
+ %rdx1.0 = fadd reassoc nnan ninf nsz arcp afn double %mul1.0, %mul1.1
%gep2.1 = getelementptr inbounds double, ptr %arg1, i64 17
%ld2.1 = load double, ptr %gep2.1, align 8
- %mul2.1 = fmul fast double %ld2.1, %ld1.1
- %rdx2.0 = fadd fast double %mul2.0, %mul2.1
+ %mul2.1 = fmul reassoc nnan ninf nsz arcp afn double %ld2.1, %ld1.1
+ %rdx2.0 = fadd reassoc nnan ninf nsz arcp afn double %mul2.0, %mul2.1
%gep1.2 = getelementptr inbounds double, ptr %arg, i64 5
%ld1.2 = load double, ptr %gep1.2, align 8
%gep0.2 = getelementptr inbounds double, ptr %arg1, i64 2
%ld0.2 = load double, ptr %gep0.2, align 8
- %mul1.2 = fmul fast double %ld0.2, %ld1.2
- %rdx1.1 = fadd fast double %rdx1.0, %mul1.2
+ %mul1.2 = fmul reassoc nnan ninf nsz arcp afn double %ld0.2, %ld1.2
+ %rdx1.1 = fadd reassoc nnan ninf nsz arcp afn double %rdx1.0, %mul1.2
%gep2.2 = getelementptr inbounds double, ptr %arg1, i64 18
%ld2.2 = load double, ptr %gep2.2, align 8
- %mul2.2 = fmul fast double %ld2.2, %ld1.2
- %rdx2.1 = fadd fast double %rdx2.0, %mul2.2
+ %mul2.2 = fmul reassoc nnan ninf nsz arcp afn double %ld2.2, %ld1.2
+ %rdx2.1 = fadd reassoc nnan ninf nsz arcp afn double %rdx2.0, %mul2.2
%gep1.3 = getelementptr inbounds double, ptr %arg, i64 7
%ld1.3 = load double, ptr %gep1.3, align 8
%gep0.3 = getelementptr inbounds double, ptr %arg1, i64 3
%ld0.3 = load double, ptr %gep0.3, align 8
- %mul1.3 = fmul fast double %ld0.3, %ld1.3
- %rdx1.2 = fadd fast double %rdx1.1, %mul1.3
+ %mul1.3 = fmul reassoc nnan ninf nsz arcp afn double %ld0.3, %ld1.3
+ %rdx1.2 = fadd reassoc nnan ninf nsz arcp afn double %rdx1.1, %mul1.3
%gep2.3 = getelementptr inbounds double, ptr %arg1, i64 19
%ld2.3 = load double, ptr %gep2.3, align 8
- %mul2.3 = fmul fast double %ld2.3, %ld1.3
- %rdx2.2 = fadd fast double %rdx2.1, %mul2.3
+ %mul2.3 = fmul reassoc nnan ninf nsz arcp afn double %ld2.3, %ld1.3
+ %rdx2.2 = fadd reassoc nnan ninf nsz arcp afn double %rdx2.1, %mul2.3
%gep1.4 = getelementptr inbounds double, ptr %arg, i64 9
%ld1.4 = load double, ptr %gep1.4, align 8
%gep0.4 = getelementptr inbounds double, ptr %arg1, i64 4
%ld0.4 = load double, ptr %gep0.4, align 8
- %mul1.4 = fmul fast double %ld0.4, %ld1.4
- %rdx1.3 = fadd fast double %rdx1.2, %mul1.4
+ %mul1.4 = fmul reassoc nnan ninf nsz arcp afn double %ld0.4, %ld1.4
+ %rdx1.3 = fadd reassoc nnan ninf nsz arcp afn double %rdx1.2, %mul1.4
%gep2.4 = getelementptr inbounds double, ptr %arg1, i64 20
%ld2.4 = load double, ptr %gep2.4, align 8
- %mul2.4 = fmul fast double %ld2.4, %ld1.4
- %rdx2.3 = fadd fast double %rdx2.2, %mul2.4
+ %mul2.4 = fmul reassoc nnan ninf nsz arcp afn double %ld2.4, %ld1.4
+ %rdx2.3 = fadd reassoc nnan ninf nsz arcp afn double %rdx2.2, %mul2.4
%gep1.5 = getelementptr inbounds double, ptr %arg, i64 11
%ld1.5 = load double, ptr %gep1.5, align 8
%gep0.5 = getelementptr inbounds double, ptr %arg1, i64 5
%ld0.5 = load double, ptr %gep0.5, align 8
- %mul1.5 = fmul fast double %ld0.5, %ld1.5
- %rdx1.4 = fadd fast double %rdx1.3, %mul1.5
+ %mul1.5 = fmul reassoc nnan ninf nsz arcp afn double %ld0.5, %ld1.5
+ %rdx1.4 = fadd reassoc nnan ninf nsz arcp afn double %rdx1.3, %mul1.5
%gep2.5 = getelementptr inbounds double, ptr %arg1, i64 21
%ld2.5 = load double, ptr %gep2.5, align 8
- %mul2.5 = fmul fast double %ld2.5, %ld1.5
- %rdx2.4 = fadd fast double %rdx2.3, %mul2.5
+ %mul2.5 = fmul reassoc nnan ninf nsz arcp afn double %ld2.5, %ld1.5
+ %rdx2.4 = fadd reassoc nnan ninf nsz arcp afn double %rdx2.3, %mul2.5
%gep1.6 = getelementptr inbounds double, ptr %arg, i64 13
%ld1.6 = load double, ptr %gep1.6, align 8
%gep0.6 = getelementptr inbounds double, ptr %arg1, i64 6
%ld0.6 = load double, ptr %gep0.6, align 8
- %mul1.6 = fmul fast double %ld0.6, %ld1.6
- %rdx1.5 = fadd fast double %rdx1.4, %mul1.6
+ %mul1.6 = fmul reassoc nnan ninf nsz arcp afn double %ld0.6, %ld1.6
+ %rdx1.5 = fadd reassoc nnan ninf nsz arcp afn double %rdx1.4, %mul1.6
%gep2.6 = getelementptr inbounds double, ptr %arg1, i64 22
%ld2.6 = load double, ptr %gep2.6, align 8
- %mul2.6 = fmul fast double %ld2.6, %ld1.6
- %rdx2.5 = fadd fast double %rdx2.4, %mul2.6
+ %mul2.6 = fmul reassoc nnan ninf nsz arcp afn double %ld2.6, %ld1.6
+ %rdx2.5 = fadd reassoc nnan ninf nsz arcp afn double %rdx2.4, %mul2.6
%gep1.7 = getelementptr inbounds double, ptr %arg, i64 15
%ld1.7 = load double, ptr %gep1.7, align 8
%gep0.7 = getelementptr inbounds double, ptr %arg1, i64 7
%ld0.7 = load double, ptr %gep0.7, align 8
- %mul1.7 = fmul fast double %ld0.7, %ld1.7
- %rdx1 = fadd fast double %rdx1.5, %mul1.7
+ %mul1.7 = fmul reassoc nnan ninf nsz arcp afn double %ld0.7, %ld1.7
+ %rdx1 = fadd reassoc nnan ninf nsz arcp afn double %rdx1.5, %mul1.7
%gep2.7 = getelementptr inbounds double, ptr %arg1, i64 23
%ld2.7 = load double, ptr %gep2.7, align 8
- %mul2.7 = fmul fast double %ld2.7, %ld1.7
- %rdx2 = fadd fast double %rdx2.5, %mul2.7
+ %mul2.7 = fmul reassoc nnan ninf nsz arcp afn double %ld2.7, %ld1.7
+ %rdx2 = fadd reassoc nnan ninf nsz arcp afn double %rdx2.5, %mul2.7
%i142 = insertelement <2 x double> poison, double %rdx1, i64 0
%i143 = insertelement <2 x double> %i142, double %rdx2, i64 1
%p = getelementptr inbounds double, ptr %arg2, <2 x i64> <i64 0, i64 16>
diff --git a/llvm/test/Transforms/SLPVectorizer/X86/select-logical-or-and-i1-vector.ll b/llvm/test/Transforms/SLPVectorizer/X86/select-logical-or-and-i1-vector.ll
index 979701609b499..623f33da6ee62 100644
--- a/llvm/test/Transforms/SLPVectorizer/X86/select-logical-or-and-i1-vector.ll
+++ b/llvm/test/Transforms/SLPVectorizer/X86/select-logical-or-and-i1-vector.ll
@@ -29,12 +29,12 @@ define void @select_logical_or_i1(ptr %dst,
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[Y1]], i64 1
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[Y2]], i64 2
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x float> [[TMP15]], float [[Y3]], i64 3
-; CHECK-NEXT: [[TMP17:%.*]] = fmul fast <4 x float> [[TMP12]], [[TMP16]]
+; CHECK-NEXT: [[TMP17:%.*]] = fmul reassoc nnan ninf nsz arcp afn <4 x float> [[TMP12]], [[TMP16]]
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> poison, float [[E0]], i64 0
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x float> [[TMP18]], float [[E1]], i64 1
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x float> [[TMP19]], float [[E2]], i64 2
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x float> [[TMP20]], float [[E3]], i64 3
-; CHECK-NEXT: [[TMP22:%.*]] = fadd fast <4 x float> [[TMP21]], [[TMP17]]
+; CHECK-NEXT: [[TMP22:%.*]] = fadd reassoc nnan ninf nsz arcp afn <4 x float> [[TMP21]], [[TMP17]]
; CHECK-NEXT: store <4 x float> [[TMP22]], ptr [[DST]], align 4
; CHECK-NEXT: ret void
;
@@ -60,15 +60,16 @@ entry:
%sel2 = select i1 %or2, float 0.000000e+00, float %hphb_val
%sel3 = select i1 %or3, float 0.000000e+00, float %hphb_val
- %mul0 = fmul fast float %sel0, %y0
- %mul1 = fmul fast float %sel1, %y1
- %mul2 = fmul fast float %sel2, %y2
- %mul3 = fmul fast float %sel3, %y3
+ ; No 'contract' on fmul/fadd: keep them from fusing into fmadd so the test exercises i1 select costing, not FMA costing.
+ %mul0 = fmul reassoc nnan ninf nsz arcp afn float %sel0, %y0
+ %mul1 = fmul reassoc nnan ninf nsz arcp afn float %sel1, %y1
+ %mul2 = fmul reassoc nnan ninf nsz arcp afn float %sel2, %y2
+ %mul3 = fmul reassoc nnan ninf nsz arcp afn float %sel3, %y3
- %res0 = fadd fast float %e0, %mul0
- %res1 = fadd fast float %e1, %mul1
- %res2 = fadd fast float %e2, %mul2
- %res3 = fadd fast float %e3, %mul3
+ %res0 = fadd reassoc nnan ninf nsz arcp afn float %e0, %mul0
+ %res1 = fadd reassoc nnan ninf nsz arcp afn float %e1, %mul1
+ %res2 = fadd reassoc nnan ninf nsz arcp afn float %e2, %mul2
+ %res3 = fadd reassoc nnan ninf nsz arcp afn float %e3, %mul3
store float %res0, ptr %dst, align 4
%p1 = getelementptr inbounds float, ptr %dst, i64 1
@@ -101,12 +102,12 @@ define void @select_logical_and_i1(ptr %dst,
; CHECK-NEXT: [[TMP14:%.*]] = insertelement <4 x float> [[TMP13]], float [[Y1]], i64 1
; CHECK-NEXT: [[TMP15:%.*]] = insertelement <4 x float> [[TMP14]], float [[Y2]], i64 2
; CHECK-NEXT: [[TMP16:%.*]] = insertelement <4 x float> [[TMP15]], float [[Y3]], i64 3
-; CHECK-NEXT: [[TMP17:%.*]] = fmul fast <4 x float> [[TMP12]], [[TMP16]]
+; CHECK-NEXT: [[TMP17:%.*]] = fmul reassoc nnan ninf nsz arcp afn <4 x float> [[TMP12]], [[TMP16]]
; CHECK-NEXT: [[TMP18:%.*]] = insertelement <4 x float> poison, float [[E0]], i64 0
; CHECK-NEXT: [[TMP19:%.*]] = insertelement <4 x float> [[TMP18]], float [[E1]], i64 1
; CHECK-NEXT: [[TMP20:%.*]] = insertelement <4 x float> [[TMP19]], float [[E2]], i64 2
; CHECK-NEXT: [[TMP21:%.*]] = insertelement <4 x float> [[TMP20]], float [[E3]], i64 3
-; CHECK-NEXT: [[TMP22:%.*]] = fadd fast <4 x float> [[TMP21]], [[TMP17]]
+; CHECK-NEXT: [[TMP22:%.*]] = fadd reassoc nnan ninf nsz arcp afn <4 x float> [[TMP21]], [[TMP17]]
; CHECK-NEXT: store <4 x float> [[TMP22]], ptr [[DST]], align 4
; CHECK-NEXT: ret void
;
@@ -132,15 +133,16 @@ entry:
%sel2 = select i1 %and2, float 0.000000e+00, float %hphb_val
%sel3 = select i1 %and3, float 0.000000e+00, float %hphb_val
- %mul0 = fmul fast float %sel0, %y0
- %mul1 = fmul fast float %sel1, %y1
- %mul2 = fmul fast float %sel2, %y2
- %mul3 = fmul fast float %sel3, %y3
+ ; No 'contract' on fmul/fadd: keep them from fusing into fmadd so the test exercises i1 select costing, not FMA costing.
+ %mul0 = fmul reassoc nnan ninf nsz arcp afn float %sel0, %y0
+ %mul1 = fmul reassoc nnan ninf nsz arcp afn float %sel1, %y1
+ %mul2 = fmul reassoc nnan ninf nsz arcp afn float %sel2, %y2
+ %mul3 = fmul reassoc nnan ninf nsz arcp afn float %sel3, %y3
- %res0 = fadd fast float %e0, %mul0
- %res1 = fadd fast float %e1, %mul1
- %res2 = fadd fast float %e2, %mul2
- %res3 = fadd fast float %e3, %mul3
+ %res0 = fadd reassoc nnan ninf nsz arcp afn float %e0, %mul0
+ %res1 = fadd reassoc nnan ninf nsz arcp afn float %e1, %mul1
+ %res2 = fadd reassoc nnan ninf nsz arcp afn float %e2, %mul2
+ %res3 = fadd reassoc nnan ninf nsz arcp afn float %e3, %mul3
store float %res0, ptr %dst, align 4
%p1 = getelementptr inbounds float, ptr %dst, i64 1
More information about the llvm-commits
mailing list