[llvm] 07ebd45 - [AArch64] Generate fmls for fneg+fmul (#215338)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Aug 25 05:40:12 PDT 2026
Author: Valeriy Savchenko
Date: 2026-08-25T13:40:07+01:00
New Revision: 07ebd450b07bc59b5c0a6c906d673913c0333315
URL: https://github.com/llvm/llvm-project/commit/07ebd450b07bc59b5c0a6c906d673913c0333315
DIFF: https://github.com/llvm/llvm-project/commit/07ebd450b07bc59b5c0a6c906d673913c0333315.diff
LOG: [AArch64] Generate fmls for fneg+fmul (#215338)
AArch64 has no vector FNMUL, so -(x * y) needs an FNEG after the FMUL.
FMLS with a -0.0 accumulator computes it in one instruction, and the
MOVI materializing -0.0 has no input dependency, so the FNEG leaves the
dependency chain.
The consumed FNEG/FMUL has to be single-use, otherwise it stays live and
the MOVI is added rather than substituted. Plain fmul only: the two
forms agree only when negation commutes with rounding, which fails for
the directed rounding modes constrained FP can run under.
Only f32 and f16 are covered; f64 -0.0 is not a MOVI immediate.
Added:
llvm/test/CodeGen/AArch64/neon-fnmul.ll
Modified:
llvm/lib/Target/AArch64/AArch64InstrInfo.td
llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll
llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64InstrInfo.td b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
index 93937f57b7d39..a614ec00959ac 100644
--- a/llvm/lib/Target/AArch64/AArch64InstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64InstrInfo.td
@@ -9447,6 +9447,88 @@ defm : FMLSIndexedAfterNegPatterns<
defm : FMLSIndexedAfterNegPatterns<
TriOpFrag<(any_fma node:$MHS, node:$RHS, node:$LHS)> >;
+// Vector FNMUL: -(x * y) -> FMLS(-0.0, x, y), by-element for a splat y.
+// AArch64 has no vector FNMUL, so -(x * y) needs an FNEG on top of the FMUL.
+// FMLS folds it in, and the MOVI has no input dependency, so the FNEG leaves
+// the dependency chain. The consumed FNEG/FMUL has to be single-use, or it
+// stays live and the MOVI is added rather than substituted.
+// Like the scalar FNMUL patterns above this changes the sign of a NaN result
+// (FMLS negates its multiplicand); LangRef leaves that sign non-deterministic.
+// Plain fmul only: the two forms agree only when negation commutes with
+// rounding, which fails for the directed modes constrained FP can run under.
+// Only f32 and f16 are covered; f64 -0.0 is not a MOVI immediate.
+let HasOneUse = 1 in {
+ def fneg_oneuse : PatFrag<(ops node:$src), (fneg node:$src)>;
+ def fmul_oneuse : PatFrag<(ops node:$lhs, node:$rhs),
+ (fmul node:$lhs, node:$rhs)>;
+}
+
+// -(Rn * Rm) in all its forms. Only the by-element patterns below look at
+// splats; they take Rm apart for the lane operand, so Rn is bound to the FNEG'd
+// operand -- the one a split FNEG+FMUL materializes anyway. Both FMUL orders
+// are listed because TableGen's commuted copies sort after these.
+def fnmul_frags : PatFrags<(ops node:$Rn, node:$Rm),
+ [(fneg (fmul_oneuse node:$Rn, node:$Rm)),
+ (fmul (fneg_oneuse node:$Rn), node:$Rm),
+ (fmul node:$Rm, (fneg_oneuse node:$Rn)),
+ // Lane slot for the FNEG'd operand, when it is the
+ // only splat.
+ (fmul node:$Rn, (fneg_oneuse node:$Rm))]>;
+
+let Predicates = [HasNEON, HasFullFP16] in {
+ def : Pat<(v8f16 (fnmul_frags (v8f16 V128:$Rn),
+ (v8f16 (AArch64duplane16 (v8f16 V128_lo:$Rm),
+ VectorIndexH:$idx)))),
+ (FMLSv8i16_indexed (MOVIv8i16 (i32 128), (i32 8)), V128:$Rn,
+ V128_lo:$Rm, VectorIndexH:$idx)>;
+ def : Pat<(v8f16 (fnmul_frags (v8f16 V128:$Rn),
+ (v8f16 (AArch64dup (f16 FPR16Op_lo:$Rm))))),
+ (FMLSv8i16_indexed (MOVIv8i16 (i32 128), (i32 8)), V128:$Rn,
+ (SUBREG_TO_REG (f16 FPR16Op_lo:$Rm), hsub),
+ (i64 0))>;
+ def : Pat<(v4f16 (fnmul_frags (v4f16 V64:$Rn),
+ (v4f16 (AArch64duplane16 (v8f16 V128_lo:$Rm),
+ VectorIndexH:$idx)))),
+ (FMLSv4i16_indexed (MOVIv4i16 (i32 128), (i32 8)), V64:$Rn,
+ V128_lo:$Rm, VectorIndexH:$idx)>;
+ def : Pat<(v4f16 (fnmul_frags (v4f16 V64:$Rn),
+ (v4f16 (AArch64dup (f16 FPR16Op_lo:$Rm))))),
+ (FMLSv4i16_indexed (MOVIv4i16 (i32 128), (i32 8)), V64:$Rn,
+ (SUBREG_TO_REG (f16 FPR16Op_lo:$Rm), hsub),
+ (i64 0))>;
+
+ def : Pat<(v8f16 (fnmul_frags (v8f16 V128:$Rn), (v8f16 V128:$Rm))),
+ (FMLSv8f16 (MOVIv8i16 (i32 128), (i32 8)), V128:$Rn, V128:$Rm)>;
+ def : Pat<(v4f16 (fnmul_frags (v4f16 V64:$Rn), (v4f16 V64:$Rm))),
+ (FMLSv4f16 (MOVIv4i16 (i32 128), (i32 8)), V64:$Rn, V64:$Rm)>;
+}
+
+let Predicates = [HasNEON] in {
+ def : Pat<(v4f32 (fnmul_frags (v4f32 V128:$Rn),
+ (v4f32 (AArch64duplane32 (v4f32 V128:$Rm),
+ VectorIndexS:$idx)))),
+ (FMLSv4i32_indexed (MOVIv4i32 (i32 128), (i32 24)), V128:$Rn,
+ V128:$Rm, VectorIndexS:$idx)>;
+ def : Pat<(v4f32 (fnmul_frags (v4f32 V128:$Rn),
+ (v4f32 (AArch64dup (f32 FPR32Op:$Rm))))),
+ (FMLSv4i32_indexed (MOVIv4i32 (i32 128), (i32 24)), V128:$Rn,
+ (SUBREG_TO_REG FPR32Op:$Rm, ssub), (i64 0))>;
+ def : Pat<(v2f32 (fnmul_frags (v2f32 V64:$Rn),
+ (v2f32 (AArch64duplane32 (v4f32 V128:$Rm),
+ VectorIndexS:$idx)))),
+ (FMLSv2i32_indexed (MOVIv2i32 (i32 128), (i32 24)), V64:$Rn,
+ V128:$Rm, VectorIndexS:$idx)>;
+ def : Pat<(v2f32 (fnmul_frags (v2f32 V64:$Rn),
+ (v2f32 (AArch64dup (f32 FPR32Op:$Rm))))),
+ (FMLSv2i32_indexed (MOVIv2i32 (i32 128), (i32 24)), V64:$Rn,
+ (SUBREG_TO_REG FPR32Op:$Rm, ssub), (i64 0))>;
+
+ def : Pat<(v4f32 (fnmul_frags (v4f32 V128:$Rn), (v4f32 V128:$Rm))),
+ (FMLSv4f32 (MOVIv4i32 (i32 128), (i32 24)), V128:$Rn, V128:$Rm)>;
+ def : Pat<(v2f32 (fnmul_frags (v2f32 V64:$Rn), (v2f32 V64:$Rm))),
+ (FMLSv2f32 (MOVIv2i32 (i32 128), (i32 24)), V64:$Rn, V64:$Rm)>;
+}
+
defm FMULX : SIMDFPIndexed<1, 0b1001, "fmulx", int_aarch64_neon_fmulx>;
defm FMUL : SIMDFPIndexed<0, 0b1001, "fmul", any_fmul>;
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
index 5970b2bb2dfd7..572412fb16ed7 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-fma.ll
@@ -30,17 +30,17 @@ define <4 x float> @complex_mul_fmuladd(<4 x float> %a, <4 x float> %b) {
define <4 x float> @complex_mul_fmuladd_no_fmf(<4 x float> %a, <4 x float> %b) {
; CHECK-LABEL: complex_mul_fmuladd_no_fmf:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov d2, v0.d[1]
-; CHECK-NEXT: mov d3, v1.d[1]
-; CHECK-NEXT: zip2 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip1 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: fmul v3.2s, v4.2s, v5.2s
-; CHECK-NEXT: fneg v2.2s, v3.2s
-; CHECK-NEXT: fmul v3.2s, v4.2s, v1.2s
-; CHECK-NEXT: fmla v2.2s, v1.2s, v0.2s
-; CHECK-NEXT: fmla v3.2s, v5.2s, v0.2s
+; CHECK-NEXT: mov d3, v0.d[1]
+; CHECK-NEXT: mov d4, v1.d[1]
+; CHECK-NEXT: movi v2.2s, #128, lsl #24
+; CHECK-NEXT: zip2 v5.2s, v0.2s, v3.2s
+; CHECK-NEXT: zip1 v6.2s, v1.2s, v4.2s
+; CHECK-NEXT: zip2 v1.2s, v1.2s, v4.2s
+; CHECK-NEXT: zip1 v0.2s, v0.2s, v3.2s
+; CHECK-NEXT: fmls v2.2s, v5.2s, v1.2s
+; CHECK-NEXT: fmul v3.2s, v5.2s, v6.2s
+; CHECK-NEXT: fmla v2.2s, v6.2s, v0.2s
+; CHECK-NEXT: fmla v3.2s, v1.2s, v0.2s
; CHECK-NEXT: zip1 v0.4s, v2.4s, v3.4s
; CHECK-NEXT: ret
%a.re = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll
index 7e0b46107ceea..36f2fe038535e 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-mixed-cases.ll
@@ -42,21 +42,21 @@ entry:
define <4 x float> @add_mul(<4 x float> %a, <4 x float> %b, <4 x float> %c) {
; CHECK-LABEL: add_mul:
; CHECK: // %bb.0: // %entry
+; CHECK-NEXT: fsub v3.4s, v1.4s, v2.4s
; CHECK-NEXT: fsub v0.4s, v1.4s, v0.4s
-; CHECK-NEXT: fsub v1.4s, v1.4s, v2.4s
-; CHECK-NEXT: mov d3, v2.d[1]
-; CHECK-NEXT: mov d4, v0.d[1]
-; CHECK-NEXT: mov d5, v1.d[1]
-; CHECK-NEXT: zip2 v0.2s, v0.2s, v4.2s
-; CHECK-NEXT: zip2 v4.2s, v2.2s, v3.2s
-; CHECK-NEXT: zip1 v1.2s, v1.2s, v5.2s
-; CHECK-NEXT: zip1 v2.2s, v2.2s, v3.2s
-; CHECK-NEXT: fmul v5.2s, v4.2s, v0.2s
-; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT: fneg v4.2s, v5.2s
-; CHECK-NEXT: fmla v3.2s, v0.2s, v2.2s
-; CHECK-NEXT: fmla v4.2s, v1.2s, v2.2s
-; CHECK-NEXT: zip1 v0.4s, v4.4s, v3.4s
+; CHECK-NEXT: mov d4, v2.d[1]
+; CHECK-NEXT: mov d1, v3.d[1]
+; CHECK-NEXT: mov d5, v0.d[1]
+; CHECK-NEXT: zip1 v1.2s, v3.2s, v1.2s
+; CHECK-NEXT: zip2 v3.2s, v2.2s, v4.2s
+; CHECK-NEXT: zip2 v0.2s, v0.2s, v5.2s
+; CHECK-NEXT: movi v5.2s, #128, lsl #24
+; CHECK-NEXT: zip1 v2.2s, v2.2s, v4.2s
+; CHECK-NEXT: fmul v4.2s, v1.2s, v3.2s
+; CHECK-NEXT: fmls v5.2s, v3.2s, v0.2s
+; CHECK-NEXT: fmla v4.2s, v0.2s, v2.2s
+; CHECK-NEXT: fmla v5.2s, v1.2s, v2.2s
+; CHECK-NEXT: zip1 v0.4s, v5.4s, v4.4s
; CHECK-NEXT: ret
entry:
%0 = fsub fast <4 x float> %b, %c
@@ -81,24 +81,24 @@ define <4 x float> @mul_mul270_mul(<4 x float> %a, <4 x float> %b, <4 x float> %
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov d3, v2.d[1]
; CHECK-NEXT: mov d4, v1.d[1]
+; CHECK-NEXT: movi v7.2s, #128, lsl #24
; CHECK-NEXT: zip1 v5.2s, v2.2s, v3.2s
; CHECK-NEXT: zip1 v6.2s, v1.2s, v4.2s
-; CHECK-NEXT: zip2 v2.2s, v2.2s, v3.2s
; CHECK-NEXT: zip2 v1.2s, v1.2s, v4.2s
-; CHECK-NEXT: mov d3, v0.d[1]
-; CHECK-NEXT: fmul v7.2s, v6.2s, v5.2s
-; CHECK-NEXT: fneg v4.2s, v7.2s
-; CHECK-NEXT: zip2 v7.2s, v0.2s, v3.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v3.2s
-; CHECK-NEXT: fmla v4.2s, v2.2s, v1.2s
-; CHECK-NEXT: fmul v1.2s, v1.2s, v5.2s
-; CHECK-NEXT: fmul v3.2s, v4.2s, v7.2s
-; CHECK-NEXT: fmla v1.2s, v2.2s, v6.2s
-; CHECK-NEXT: fmul v2.2s, v4.2s, v0.2s
-; CHECK-NEXT: fneg v3.2s, v3.2s
-; CHECK-NEXT: fmla v2.2s, v7.2s, v1.2s
-; CHECK-NEXT: fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT: zip1 v0.4s, v3.4s, v2.4s
+; CHECK-NEXT: mov d4, v0.d[1]
+; CHECK-NEXT: zip2 v2.2s, v2.2s, v3.2s
+; CHECK-NEXT: fmls v7.2s, v6.2s, v5.2s
+; CHECK-NEXT: fmul v3.2s, v1.2s, v5.2s
+; CHECK-NEXT: zip1 v5.2s, v0.2s, v4.2s
+; CHECK-NEXT: zip2 v0.2s, v0.2s, v4.2s
+; CHECK-NEXT: movi v4.2s, #128, lsl #24
+; CHECK-NEXT: fmla v7.2s, v2.2s, v1.2s
+; CHECK-NEXT: fmla v3.2s, v2.2s, v6.2s
+; CHECK-NEXT: fmul v1.2s, v7.2s, v5.2s
+; CHECK-NEXT: fmls v4.2s, v7.2s, v0.2s
+; CHECK-NEXT: fmla v1.2s, v0.2s, v3.2s
+; CHECK-NEXT: fmla v4.2s, v5.2s, v3.2s
+; CHECK-NEXT: zip1 v0.4s, v4.4s, v1.4s
; CHECK-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %c, <4 x float> poison, <2 x i32> <i32 0, i32 2>
@@ -311,25 +311,25 @@ entry:
define <4 x float> @mul_triangle_multiuses(<4 x float> %a, <4 x float> %b, ptr %p) {
; CHECK-LABEL: mul_triangle_multiuses:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov d2, v0.d[1]
-; CHECK-NEXT: mov d3, v1.d[1]
-; CHECK-NEXT: zip2 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: fmul v2.2s, v4.2s, v5.2s
-; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT: fmla v2.2s, v0.2s, v1.2s
-; CHECK-NEXT: fneg v1.2s, v3.2s
-; CHECK-NEXT: fmul v3.2s, v2.2s, v4.2s
-; CHECK-NEXT: fmla v1.2s, v0.2s, v5.2s
-; CHECK-NEXT: fmul v5.2s, v2.2s, v0.2s
-; CHECK-NEXT: fneg v3.2s, v3.2s
-; CHECK-NEXT: fmla v5.2s, v4.2s, v1.2s
+; CHECK-NEXT: mov d3, v0.d[1]
+; CHECK-NEXT: mov d4, v1.d[1]
+; CHECK-NEXT: movi v2.2s, #128, lsl #24
+; CHECK-NEXT: zip2 v5.2s, v0.2s, v3.2s
+; CHECK-NEXT: zip1 v6.2s, v1.2s, v4.2s
+; CHECK-NEXT: zip1 v0.2s, v0.2s, v3.2s
+; CHECK-NEXT: zip2 v1.2s, v1.2s, v4.2s
+; CHECK-NEXT: fmul v3.2s, v5.2s, v6.2s
+; CHECK-NEXT: fmls v2.2s, v1.2s, v5.2s
; CHECK-NEXT: fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT: mov v1.d[1], v2.d[0]
-; CHECK-NEXT: zip1 v0.4s, v3.4s, v5.4s
-; CHECK-NEXT: str q1, [x0]
+; CHECK-NEXT: movi v1.2s, #128, lsl #24
+; CHECK-NEXT: fmla v2.2s, v0.2s, v6.2s
+; CHECK-NEXT: fmul v4.2s, v3.2s, v0.2s
+; CHECK-NEXT: fmls v1.2s, v3.2s, v5.2s
+; CHECK-NEXT: fmla v4.2s, v5.2s, v2.2s
+; CHECK-NEXT: fmla v1.2s, v0.2s, v2.2s
+; CHECK-NEXT: mov v2.d[1], v3.d[0]
+; CHECK-NEXT: zip1 v0.4s, v1.4s, v4.4s
+; CHECK-NEXT: str q2, [x0]
; CHECK-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
@@ -444,19 +444,19 @@ define <4 x float> @mul_divequal(<4 x float> %a, <4 x float> %b, <4 x float> %c)
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: mov d3, v0.d[1]
; CHECK-NEXT: mov d4, v1.d[1]
-; CHECK-NEXT: zip2 v5.2s, v0.2s, v3.2s
-; CHECK-NEXT: zip2 v6.2s, v1.2s, v4.2s
+; CHECK-NEXT: movi v5.2s, #128, lsl #24
+; CHECK-NEXT: zip2 v6.2s, v0.2s, v3.2s
+; CHECK-NEXT: zip2 v7.2s, v1.2s, v4.2s
; CHECK-NEXT: zip1 v0.2s, v0.2s, v3.2s
; CHECK-NEXT: zip1 v1.2s, v1.2s, v4.2s
; CHECK-NEXT: mov d3, v2.d[1]
-; CHECK-NEXT: fmul v7.2s, v5.2s, v6.2s
-; CHECK-NEXT: fneg v4.2s, v7.2s
-; CHECK-NEXT: zip1 v7.2s, v2.2s, v3.2s
+; CHECK-NEXT: fmls v5.2s, v6.2s, v7.2s
+; CHECK-NEXT: zip1 v4.2s, v2.2s, v3.2s
; CHECK-NEXT: zip2 v2.2s, v2.2s, v3.2s
-; CHECK-NEXT: fmla v4.2s, v0.2s, v1.2s
-; CHECK-NEXT: fmul v0.2s, v6.2s, v0.2s
-; CHECK-NEXT: fmla v0.2s, v5.2s, v1.2s
-; CHECK-NEXT: fdiv v4.2s, v4.2s, v7.2s
+; CHECK-NEXT: fmla v5.2s, v0.2s, v1.2s
+; CHECK-NEXT: fmul v0.2s, v7.2s, v0.2s
+; CHECK-NEXT: fmla v0.2s, v6.2s, v1.2s
+; CHECK-NEXT: fdiv v4.2s, v5.2s, v4.2s
; CHECK-NEXT: fdiv v0.2s, v0.2s, v2.2s
; CHECK-NEXT: zip1 v0.4s, v4.4s, v0.4s
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
index 434fb08c3d4b6..0e4b3eee1f991 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-multiuses.ll
@@ -47,24 +47,24 @@ entry:
define <4 x float> @mul_triangle_external_use(<4 x float> %a, <4 x float> %b, ptr %p) {
; CHECK-LABEL: mul_triangle_external_use:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov d2, v0.d[1]
-; CHECK-NEXT: mov d3, v1.d[1]
-; CHECK-NEXT: zip2 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: fmul v2.2s, v4.2s, v5.2s
-; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT: fmla v2.2s, v0.2s, v1.2s
-; CHECK-NEXT: fneg v1.2s, v3.2s
-; CHECK-NEXT: fmul v3.2s, v2.2s, v4.2s
-; CHECK-NEXT: str d2, [x0]
-; CHECK-NEXT: fmla v1.2s, v0.2s, v5.2s
-; CHECK-NEXT: fmul v5.2s, v2.2s, v0.2s
-; CHECK-NEXT: fneg v3.2s, v3.2s
-; CHECK-NEXT: fmla v5.2s, v4.2s, v1.2s
+; CHECK-NEXT: mov d3, v0.d[1]
+; CHECK-NEXT: mov d4, v1.d[1]
+; CHECK-NEXT: movi v2.2s, #128, lsl #24
+; CHECK-NEXT: zip2 v5.2s, v0.2s, v3.2s
+; CHECK-NEXT: zip1 v6.2s, v1.2s, v4.2s
+; CHECK-NEXT: zip1 v0.2s, v0.2s, v3.2s
+; CHECK-NEXT: zip2 v1.2s, v1.2s, v4.2s
+; CHECK-NEXT: fmul v3.2s, v5.2s, v6.2s
+; CHECK-NEXT: fmls v2.2s, v1.2s, v5.2s
; CHECK-NEXT: fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT: zip1 v0.4s, v3.4s, v5.4s
+; CHECK-NEXT: movi v1.2s, #128, lsl #24
+; CHECK-NEXT: fmla v2.2s, v0.2s, v6.2s
+; CHECK-NEXT: fmul v4.2s, v3.2s, v0.2s
+; CHECK-NEXT: fmls v1.2s, v3.2s, v5.2s
+; CHECK-NEXT: str d3, [x0]
+; CHECK-NEXT: fmla v4.2s, v5.2s, v2.2s
+; CHECK-NEXT: fmla v1.2s, v0.2s, v2.2s
+; CHECK-NEXT: zip1 v0.4s, v1.4s, v4.4s
; CHECK-NEXT: ret
entry:
%strided.vec = shufflevector <4 x float> %a, <4 x float> poison, <2 x i32> <i32 0, i32 2>
@@ -95,58 +95,58 @@ entry:
define <4 x float> @multiple_muls_shuffle_external(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, ptr %p1, ptr %p2) {
; CHECK-IAENABLED-LABEL: multiple_muls_shuffle_external:
; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov d4, v1.d[1]
; CHECK-IAENABLED-NEXT: mov d5, v0.d[1]
-; CHECK-IAENABLED-NEXT: mov d6, v1.d[1]
-; CHECK-IAENABLED-NEXT: mov d4, v2.d[1]
+; CHECK-IAENABLED-NEXT: mov d16, v2.d[1]
+; CHECK-IAENABLED-NEXT: movi v17.2s, #128, lsl #24
+; CHECK-IAENABLED-NEXT: zip1 v6.2s, v1.2s, v4.2s
; CHECK-IAENABLED-NEXT: zip2 v7.2s, v0.2s, v5.2s
-; CHECK-IAENABLED-NEXT: zip1 v16.2s, v1.2s, v6.2s
-; CHECK-IAENABLED-NEXT: zip2 v1.2s, v1.2s, v6.2s
; CHECK-IAENABLED-NEXT: zip1 v0.2s, v0.2s, v5.2s
-; CHECK-IAENABLED-NEXT: fmul v5.2s, v16.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fmul v6.2s, v1.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fmla v5.2s, v0.2s, v1.2s
-; CHECK-IAENABLED-NEXT: fneg v1.2s, v6.2s
-; CHECK-IAENABLED-NEXT: zip1 v6.2s, v2.2s, v4.2s
-; CHECK-IAENABLED-NEXT: zip2 v4.2s, v2.2s, v4.2s
-; CHECK-IAENABLED-NEXT: fmla v1.2s, v0.2s, v16.2s
-; CHECK-IAENABLED-NEXT: fmul v17.2s, v6.2s, v5.2s
+; CHECK-IAENABLED-NEXT: zip2 v1.2s, v1.2s, v4.2s
+; CHECK-IAENABLED-NEXT: movi v5.2s, #128, lsl #24
+; CHECK-IAENABLED-NEXT: fmul v4.2s, v6.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmls v5.2s, v1.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmla v4.2s, v0.2s, v1.2s
+; CHECK-IAENABLED-NEXT: zip1 v1.2s, v2.2s, v16.2s
+; CHECK-IAENABLED-NEXT: fmla v5.2s, v0.2s, v6.2s
+; CHECK-IAENABLED-NEXT: zip2 v6.2s, v2.2s, v16.2s
; CHECK-IAENABLED-NEXT: movi v0.2d, #0000000000000000
-; CHECK-IAENABLED-NEXT: fmul v5.2s, v4.2s, v5.2s
-; CHECK-IAENABLED-NEXT: fmla v17.2s, v1.2s, v4.2s
+; CHECK-IAENABLED-NEXT: fmul v18.2s, v1.2s, v4.2s
; CHECK-IAENABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
-; CHECK-IAENABLED-NEXT: str d1, [x0]
-; CHECK-IAENABLED-NEXT: fneg v16.2s, v5.2s
+; CHECK-IAENABLED-NEXT: str d5, [x0]
+; CHECK-IAENABLED-NEXT: fmla v18.2s, v5.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmls v17.2s, v6.2s, v4.2s
; CHECK-IAENABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
-; CHECK-IAENABLED-NEXT: fmla v16.2s, v1.2s, v6.2s
-; CHECK-IAENABLED-NEXT: st2 { v16.2s, v17.2s }, [x1]
+; CHECK-IAENABLED-NEXT: fmla v17.2s, v5.2s, v1.2s
+; CHECK-IAENABLED-NEXT: st2 { v17.2s, v18.2s }, [x1]
; CHECK-IAENABLED-NEXT: ret
;
; CHECK-IADISABLED-LABEL: multiple_muls_shuffle_external:
; CHECK-IADISABLED: // %bb.0: // %entry
-; CHECK-IADISABLED-NEXT: mov d5, v0.d[1]
-; CHECK-IADISABLED-NEXT: mov d6, v1.d[1]
-; CHECK-IADISABLED-NEXT: mov d4, v2.d[1]
-; CHECK-IADISABLED-NEXT: zip2 v7.2s, v0.2s, v5.2s
-; CHECK-IADISABLED-NEXT: zip1 v16.2s, v1.2s, v6.2s
-; CHECK-IADISABLED-NEXT: zip1 v5.2s, v0.2s, v5.2s
-; CHECK-IADISABLED-NEXT: zip2 v0.2s, v1.2s, v6.2s
-; CHECK-IADISABLED-NEXT: zip2 v6.2s, v2.2s, v4.2s
-; CHECK-IADISABLED-NEXT: zip1 v4.2s, v2.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fmul v1.2s, v16.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fmla v1.2s, v5.2s, v0.2s
-; CHECK-IADISABLED-NEXT: fmul v0.2s, v0.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fneg v7.2s, v0.2s
-; CHECK-IADISABLED-NEXT: fmul v17.2s, v6.2s, v1.2s
+; CHECK-IADISABLED-NEXT: mov d4, v0.d[1]
+; CHECK-IADISABLED-NEXT: mov d5, v1.d[1]
+; CHECK-IADISABLED-NEXT: mov d16, v2.d[1]
+; CHECK-IADISABLED-NEXT: movi v17.2s, #128, lsl #24
+; CHECK-IADISABLED-NEXT: zip2 v6.2s, v0.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip1 v7.2s, v1.2s, v5.2s
+; CHECK-IADISABLED-NEXT: zip1 v4.2s, v0.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip2 v0.2s, v1.2s, v5.2s
+; CHECK-IADISABLED-NEXT: movi v5.2s, #128, lsl #24
+; CHECK-IADISABLED-NEXT: fmul v1.2s, v7.2s, v6.2s
+; CHECK-IADISABLED-NEXT: fmls v5.2s, v0.2s, v6.2s
+; CHECK-IADISABLED-NEXT: zip1 v6.2s, v2.2s, v16.2s
+; CHECK-IADISABLED-NEXT: zip2 v16.2s, v2.2s, v16.2s
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v4.2s, v0.2s
; CHECK-IADISABLED-NEXT: movi v0.2d, #0000000000000000
-; CHECK-IADISABLED-NEXT: fmul v1.2s, v4.2s, v1.2s
-; CHECK-IADISABLED-NEXT: fmla v7.2s, v5.2s, v16.2s
-; CHECK-IADISABLED-NEXT: fneg v5.2s, v17.2s
+; CHECK-IADISABLED-NEXT: fmla v5.2s, v4.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fmul v4.2s, v6.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmls v17.2s, v16.2s, v1.2s
; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
-; CHECK-IADISABLED-NEXT: fmla v1.2s, v7.2s, v6.2s
-; CHECK-IADISABLED-NEXT: fmla v5.2s, v7.2s, v4.2s
+; CHECK-IADISABLED-NEXT: str d5, [x0]
+; CHECK-IADISABLED-NEXT: fmla v4.2s, v5.2s, v16.2s
+; CHECK-IADISABLED-NEXT: fmla v17.2s, v5.2s, v6.2s
; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
-; CHECK-IADISABLED-NEXT: str d7, [x0]
-; CHECK-IADISABLED-NEXT: zip1 v1.4s, v5.4s, v1.4s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v17.4s, v4.4s
; CHECK-IADISABLED-NEXT: str q1, [x1]
; CHECK-IADISABLED-NEXT: ret
entry:
@@ -190,24 +190,24 @@ define <4 x float> @multiple_muls_shuffle_external_with_loads(ptr %ptr_a, ptr %p
; CHECK-IAENABLED-LABEL: multiple_muls_shuffle_external_with_loads:
; CHECK-IAENABLED: // %bb.0: // %entry
; CHECK-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-IAENABLED-NEXT: movi v4.2s, #128, lsl #24
; CHECK-IAENABLED-NEXT: ld2 { v2.2s, v3.2s }, [x1]
-; CHECK-IAENABLED-NEXT: fmul v4.2s, v3.2s, v1.2s
+; CHECK-IAENABLED-NEXT: fmls v4.2s, v3.2s, v1.2s
; CHECK-IAENABLED-NEXT: fmul v6.2s, v2.2s, v1.2s
-; CHECK-IAENABLED-NEXT: fneg v4.2s, v4.2s
-; CHECK-IAENABLED-NEXT: fmla v6.2s, v0.2s, v3.2s
; CHECK-IAENABLED-NEXT: fmla v4.2s, v0.2s, v2.2s
+; CHECK-IAENABLED-NEXT: fmla v6.2s, v0.2s, v3.2s
+; CHECK-IAENABLED-NEXT: movi v1.2s, #128, lsl #24
; CHECK-IAENABLED-NEXT: str d4, [x4]
; CHECK-IAENABLED-NEXT: ldr q5, [x2]
; CHECK-IAENABLED-NEXT: mov d7, v5.d[1]
; CHECK-IAENABLED-NEXT: zip1 v0.2s, v5.2s, v7.2s
-; CHECK-IAENABLED-NEXT: zip2 v1.2s, v5.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fmul v3.2s, v0.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmul v6.2s, v1.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmla v3.2s, v4.2s, v1.2s
-; CHECK-IAENABLED-NEXT: fneg v2.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmla v2.2s, v4.2s, v0.2s
+; CHECK-IAENABLED-NEXT: zip2 v3.2s, v5.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmul v2.2s, v0.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmla v2.2s, v4.2s, v3.2s
+; CHECK-IAENABLED-NEXT: fmls v1.2s, v3.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmla v1.2s, v4.2s, v0.2s
; CHECK-IAENABLED-NEXT: movi v0.2d, #0000000000000000
-; CHECK-IAENABLED-NEXT: st2 { v2.2s, v3.2s }, [x5]
+; CHECK-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x5]
; CHECK-IAENABLED-NEXT: ldr q1, [x3]
; CHECK-IAENABLED-NEXT: fcmla v0.4s, v5.4s, v1.4s, #0
; CHECK-IAENABLED-NEXT: fcmla v0.4s, v5.4s, v1.4s, #90
@@ -217,33 +217,33 @@ define <4 x float> @multiple_muls_shuffle_external_with_loads(ptr %ptr_a, ptr %p
; CHECK-IADISABLED: // %bb.0: // %entry
; CHECK-IADISABLED-NEXT: ldr q0, [x0]
; CHECK-IADISABLED-NEXT: ldr q1, [x1]
+; CHECK-IADISABLED-NEXT: movi v6.2s, #128, lsl #24
; CHECK-IADISABLED-NEXT: mov d2, v0.d[1]
; CHECK-IADISABLED-NEXT: mov d3, v1.d[1]
; CHECK-IADISABLED-NEXT: zip2 v4.2s, v0.2s, v2.2s
; CHECK-IADISABLED-NEXT: zip2 v5.2s, v1.2s, v3.2s
; CHECK-IADISABLED-NEXT: zip1 v0.2s, v0.2s, v2.2s
; CHECK-IADISABLED-NEXT: zip1 v1.2s, v1.2s, v3.2s
-; CHECK-IADISABLED-NEXT: fmul v6.2s, v5.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fneg v2.2s, v6.2s
-; CHECK-IADISABLED-NEXT: fmla v2.2s, v0.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmls v6.2s, v5.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmla v6.2s, v0.2s, v1.2s
; CHECK-IADISABLED-NEXT: fmul v1.2s, v1.2s, v4.2s
-; CHECK-IADISABLED-NEXT: str d2, [x4]
+; CHECK-IADISABLED-NEXT: str d6, [x4]
; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v5.2s
-; CHECK-IADISABLED-NEXT: ldr q3, [x2]
-; CHECK-IADISABLED-NEXT: mov d4, v3.d[1]
-; CHECK-IADISABLED-NEXT: zip2 v0.2s, v3.2s, v4.2s
-; CHECK-IADISABLED-NEXT: zip1 v4.2s, v3.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fmul v5.2s, v0.2s, v1.2s
-; CHECK-IADISABLED-NEXT: fmul v1.2s, v4.2s, v1.2s
-; CHECK-IADISABLED-NEXT: fneg v5.2s, v5.2s
-; CHECK-IADISABLED-NEXT: fmla v1.2s, v2.2s, v0.2s
+; CHECK-IADISABLED-NEXT: movi v0.2s, #128, lsl #24
+; CHECK-IADISABLED-NEXT: ldr q2, [x2]
+; CHECK-IADISABLED-NEXT: mov d3, v2.d[1]
+; CHECK-IADISABLED-NEXT: zip1 v4.2s, v2.2s, v3.2s
+; CHECK-IADISABLED-NEXT: zip2 v3.2s, v2.2s, v3.2s
+; CHECK-IADISABLED-NEXT: fmul v5.2s, v4.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmls v0.2s, v3.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmla v5.2s, v6.2s, v3.2s
+; CHECK-IADISABLED-NEXT: fmla v0.2s, v6.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v0.4s, v5.4s
; CHECK-IADISABLED-NEXT: movi v0.2d, #0000000000000000
-; CHECK-IADISABLED-NEXT: fmla v5.2s, v2.2s, v4.2s
-; CHECK-IADISABLED-NEXT: zip1 v1.4s, v5.4s, v1.4s
; CHECK-IADISABLED-NEXT: str q1, [x5]
; CHECK-IADISABLED-NEXT: ldr q1, [x3]
-; CHECK-IADISABLED-NEXT: fcmla v0.4s, v3.4s, v1.4s, #0
-; CHECK-IADISABLED-NEXT: fcmla v0.4s, v3.4s, v1.4s, #90
+; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v1.4s, #0
+; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v1.4s, #90
; CHECK-IADISABLED-NEXT: ret
entry:
%a = load <4 x float>, ptr %ptr_a
@@ -292,34 +292,34 @@ define <4 x float> @multiple_muls_mul_external(<4 x float> %a, <4 x float> %b, <
; CHECK-IAENABLED: // %bb.0: // %entry
; CHECK-IAENABLED-NEXT: mov d4, v0.d[1]
; CHECK-IAENABLED-NEXT: mov d5, v1.d[1]
-; CHECK-IAENABLED-NEXT: mov d16, v2.d[1]
-; CHECK-IAENABLED-NEXT: mov d17, v3.d[1]
-; CHECK-IAENABLED-NEXT: zip2 v6.2s, v0.2s, v4.2s
-; CHECK-IAENABLED-NEXT: zip2 v7.2s, v1.2s, v5.2s
-; CHECK-IAENABLED-NEXT: zip1 v19.2s, v2.2s, v16.2s
-; CHECK-IAENABLED-NEXT: zip2 v2.2s, v2.2s, v16.2s
-; CHECK-IAENABLED-NEXT: zip2 v16.2s, v3.2s, v17.2s
+; CHECK-IAENABLED-NEXT: mov d6, v2.d[1]
+; CHECK-IAENABLED-NEXT: mov d7, v3.d[1]
+; CHECK-IAENABLED-NEXT: movi v16.2s, #128, lsl #24
+; CHECK-IAENABLED-NEXT: zip2 v17.2s, v0.2s, v4.2s
+; CHECK-IAENABLED-NEXT: zip2 v18.2s, v1.2s, v5.2s
+; CHECK-IAENABLED-NEXT: zip1 v19.2s, v2.2s, v6.2s
+; CHECK-IAENABLED-NEXT: zip2 v20.2s, v3.2s, v7.2s
; CHECK-IAENABLED-NEXT: zip1 v0.2s, v0.2s, v4.2s
; CHECK-IAENABLED-NEXT: zip1 v1.2s, v1.2s, v5.2s
-; CHECK-IAENABLED-NEXT: zip1 v3.2s, v3.2s, v17.2s
-; CHECK-IAENABLED-NEXT: fmul v18.2s, v6.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fmul v5.2s, v19.2s, v16.2s
-; CHECK-IAENABLED-NEXT: fmul v16.2s, v2.2s, v16.2s
-; CHECK-IAENABLED-NEXT: fmul v7.2s, v0.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fneg v4.2s, v18.2s
+; CHECK-IAENABLED-NEXT: zip2 v2.2s, v2.2s, v6.2s
+; CHECK-IAENABLED-NEXT: movi v4.2s, #128, lsl #24
+; CHECK-IAENABLED-NEXT: zip1 v3.2s, v3.2s, v7.2s
+; CHECK-IAENABLED-NEXT: movi v6.2s, #128, lsl #24
+; CHECK-IAENABLED-NEXT: fmls v16.2s, v17.2s, v18.2s
+; CHECK-IAENABLED-NEXT: fmul v5.2s, v19.2s, v20.2s
+; CHECK-IAENABLED-NEXT: fmls v4.2s, v2.2s, v20.2s
+; CHECK-IAENABLED-NEXT: fmla v16.2s, v1.2s, v0.2s
+; CHECK-IAENABLED-NEXT: fmul v0.2s, v0.2s, v18.2s
; CHECK-IAENABLED-NEXT: fmla v5.2s, v3.2s, v2.2s
-; CHECK-IAENABLED-NEXT: fneg v2.2s, v16.2s
-; CHECK-IAENABLED-NEXT: fmla v7.2s, v1.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmla v4.2s, v1.2s, v0.2s
-; CHECK-IAENABLED-NEXT: fmla v2.2s, v3.2s, v19.2s
-; CHECK-IAENABLED-NEXT: fmul v0.2s, v7.2s, v5.2s
-; CHECK-IAENABLED-NEXT: fmul v17.2s, v4.2s, v5.2s
-; CHECK-IAENABLED-NEXT: str d4, [x0]
-; CHECK-IAENABLED-NEXT: fmla v17.2s, v2.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fneg v16.2s, v0.2s
-; CHECK-IAENABLED-NEXT: zip1 v0.4s, v2.4s, v5.4s
-; CHECK-IAENABLED-NEXT: fmla v16.2s, v2.2s, v4.2s
-; CHECK-IAENABLED-NEXT: st2 { v16.2s, v17.2s }, [x1]
+; CHECK-IAENABLED-NEXT: fmla v4.2s, v3.2s, v19.2s
+; CHECK-IAENABLED-NEXT: fmla v0.2s, v1.2s, v17.2s
+; CHECK-IAENABLED-NEXT: str d16, [x0]
+; CHECK-IAENABLED-NEXT: fmul v7.2s, v16.2s, v5.2s
+; CHECK-IAENABLED-NEXT: fmla v7.2s, v4.2s, v0.2s
+; CHECK-IAENABLED-NEXT: fmls v6.2s, v0.2s, v5.2s
+; CHECK-IAENABLED-NEXT: zip1 v0.4s, v4.4s, v5.4s
+; CHECK-IAENABLED-NEXT: fmla v6.2s, v4.2s, v16.2s
+; CHECK-IAENABLED-NEXT: st2 { v6.2s, v7.2s }, [x1]
; CHECK-IAENABLED-NEXT: ret
;
; CHECK-IADISABLED-LABEL: multiple_muls_mul_external:
@@ -328,32 +328,32 @@ define <4 x float> @multiple_muls_mul_external(<4 x float> %a, <4 x float> %b, <
; CHECK-IADISABLED-NEXT: mov d5, v1.d[1]
; CHECK-IADISABLED-NEXT: mov d6, v2.d[1]
; CHECK-IADISABLED-NEXT: mov d7, v3.d[1]
-; CHECK-IADISABLED-NEXT: zip1 v16.2s, v0.2s, v4.2s
-; CHECK-IADISABLED-NEXT: zip2 v17.2s, v1.2s, v5.2s
+; CHECK-IADISABLED-NEXT: movi v16.2s, #128, lsl #24
+; CHECK-IADISABLED-NEXT: zip1 v17.2s, v0.2s, v4.2s
; CHECK-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v4.2s
-; CHECK-IADISABLED-NEXT: zip1 v4.2s, v2.2s, v6.2s
-; CHECK-IADISABLED-NEXT: zip2 v18.2s, v3.2s, v7.2s
+; CHECK-IADISABLED-NEXT: zip2 v4.2s, v1.2s, v5.2s
+; CHECK-IADISABLED-NEXT: zip1 v18.2s, v2.2s, v6.2s
+; CHECK-IADISABLED-NEXT: zip2 v19.2s, v3.2s, v7.2s
; CHECK-IADISABLED-NEXT: zip1 v1.2s, v1.2s, v5.2s
; CHECK-IADISABLED-NEXT: zip2 v2.2s, v2.2s, v6.2s
; CHECK-IADISABLED-NEXT: zip1 v3.2s, v3.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fmul v5.2s, v16.2s, v17.2s
-; CHECK-IADISABLED-NEXT: fmul v6.2s, v0.2s, v17.2s
-; CHECK-IADISABLED-NEXT: fmul v7.2s, v4.2s, v18.2s
+; CHECK-IADISABLED-NEXT: fmul v5.2s, v17.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmls v16.2s, v0.2s, v4.2s
+; CHECK-IADISABLED-NEXT: movi v4.2s, #128, lsl #24
+; CHECK-IADISABLED-NEXT: fmul v6.2s, v18.2s, v19.2s
; CHECK-IADISABLED-NEXT: fmla v5.2s, v1.2s, v0.2s
-; CHECK-IADISABLED-NEXT: fmul v0.2s, v2.2s, v18.2s
-; CHECK-IADISABLED-NEXT: fmla v7.2s, v3.2s, v2.2s
-; CHECK-IADISABLED-NEXT: fneg v2.2s, v6.2s
-; CHECK-IADISABLED-NEXT: fneg v0.2s, v0.2s
-; CHECK-IADISABLED-NEXT: fmla v2.2s, v1.2s, v16.2s
-; CHECK-IADISABLED-NEXT: fmul v1.2s, v5.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fmla v0.2s, v3.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fmul v3.2s, v2.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fneg v1.2s, v1.2s
-; CHECK-IADISABLED-NEXT: str d2, [x0]
-; CHECK-IADISABLED-NEXT: fmla v3.2s, v0.2s, v5.2s
-; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v2.2s
-; CHECK-IADISABLED-NEXT: zip1 v0.4s, v0.4s, v7.4s
-; CHECK-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v3.4s
+; CHECK-IADISABLED-NEXT: fmla v16.2s, v1.2s, v17.2s
+; CHECK-IADISABLED-NEXT: movi v1.2s, #128, lsl #24
+; CHECK-IADISABLED-NEXT: fmla v6.2s, v3.2s, v2.2s
+; CHECK-IADISABLED-NEXT: fmls v4.2s, v2.2s, v19.2s
+; CHECK-IADISABLED-NEXT: str d16, [x0]
+; CHECK-IADISABLED-NEXT: fmla v4.2s, v3.2s, v18.2s
+; CHECK-IADISABLED-NEXT: fmul v2.2s, v16.2s, v6.2s
+; CHECK-IADISABLED-NEXT: fmls v1.2s, v5.2s, v6.2s
+; CHECK-IADISABLED-NEXT: fmla v2.2s, v4.2s, v5.2s
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v4.2s, v16.2s
+; CHECK-IADISABLED-NEXT: zip1 v0.4s, v4.4s, v6.4s
+; CHECK-IADISABLED-NEXT: zip1 v1.4s, v1.4s, v2.4s
; CHECK-IADISABLED-NEXT: str q1, [x1]
; CHECK-IADISABLED-NEXT: ret
entry:
@@ -522,26 +522,26 @@ entry:
define <4 x float> @mul_triangle_external_use_intrinsic(<4 x float> %a, <4 x float> %b, ptr %p) {
; CHECK-LABEL: mul_triangle_external_use_intrinsic:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: mov d2, v0.d[1]
-; CHECK-NEXT: mov d3, v1.d[1]
-; CHECK-NEXT: zip2 v4.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip1 v5.2s, v1.2s, v3.2s
-; CHECK-NEXT: zip1 v0.2s, v0.2s, v2.2s
-; CHECK-NEXT: zip2 v1.2s, v1.2s, v3.2s
-; CHECK-NEXT: fmul v2.2s, v4.2s, v5.2s
-; CHECK-NEXT: fmul v3.2s, v1.2s, v4.2s
-; CHECK-NEXT: fmla v2.2s, v0.2s, v1.2s
-; CHECK-NEXT: fneg v1.2s, v3.2s
-; CHECK-NEXT: fmul v3.2s, v2.2s, v4.2s
-; CHECK-NEXT: str d2, [x0]
-; CHECK-NEXT: fmla v1.2s, v0.2s, v5.2s
-; CHECK-NEXT: fmul v5.2s, v2.2s, v0.2s
-; CHECK-NEXT: fneg v3.2s, v3.2s
-; CHECK-NEXT: fmla v5.2s, v4.2s, v1.2s
+; CHECK-NEXT: mov d3, v0.d[1]
+; CHECK-NEXT: mov d4, v1.d[1]
+; CHECK-NEXT: movi v2.2s, #128, lsl #24
+; CHECK-NEXT: zip2 v5.2s, v0.2s, v3.2s
+; CHECK-NEXT: zip1 v6.2s, v1.2s, v4.2s
+; CHECK-NEXT: zip1 v0.2s, v0.2s, v3.2s
+; CHECK-NEXT: zip2 v1.2s, v1.2s, v4.2s
+; CHECK-NEXT: fmul v3.2s, v5.2s, v6.2s
+; CHECK-NEXT: fmls v2.2s, v1.2s, v5.2s
; CHECK-NEXT: fmla v3.2s, v0.2s, v1.2s
-; CHECK-NEXT: mov v3.d[1], v5.d[0]
-; CHECK-NEXT: rev64 v0.4s, v3.4s
-; CHECK-NEXT: uzp1 v0.4s, v3.4s, v0.4s
+; CHECK-NEXT: movi v1.2s, #128, lsl #24
+; CHECK-NEXT: fmla v2.2s, v0.2s, v6.2s
+; CHECK-NEXT: fmul v4.2s, v3.2s, v0.2s
+; CHECK-NEXT: fmls v1.2s, v3.2s, v5.2s
+; CHECK-NEXT: str d3, [x0]
+; CHECK-NEXT: fmla v4.2s, v5.2s, v2.2s
+; CHECK-NEXT: fmla v1.2s, v0.2s, v2.2s
+; CHECK-NEXT: mov v1.d[1], v4.d[0]
+; CHECK-NEXT: rev64 v0.4s, v1.4s
+; CHECK-NEXT: uzp1 v0.4s, v1.4s, v0.4s
; CHECK-NEXT: ret
entry:
%a.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %a)
@@ -571,61 +571,61 @@ entry:
define <4 x float> @multiple_muls_shuffle_external_intrinsic(<4 x float> %a, <4 x float> %b, <4 x float> %c, <4 x float> %d, ptr %p1, ptr %p2) {
; CHECK-IAENABLED-LABEL: multiple_muls_shuffle_external_intrinsic:
; CHECK-IAENABLED: // %bb.0: // %entry
+; CHECK-IAENABLED-NEXT: mov d4, v1.d[1]
; CHECK-IAENABLED-NEXT: mov d5, v0.d[1]
-; CHECK-IAENABLED-NEXT: mov d6, v1.d[1]
-; CHECK-IAENABLED-NEXT: mov d4, v2.d[1]
+; CHECK-IAENABLED-NEXT: mov d16, v2.d[1]
+; CHECK-IAENABLED-NEXT: movi v17.2s, #128, lsl #24
+; CHECK-IAENABLED-NEXT: zip1 v6.2s, v1.2s, v4.2s
; CHECK-IAENABLED-NEXT: zip2 v7.2s, v0.2s, v5.2s
-; CHECK-IAENABLED-NEXT: zip2 v16.2s, v1.2s, v6.2s
-; CHECK-IAENABLED-NEXT: zip1 v1.2s, v1.2s, v6.2s
; CHECK-IAENABLED-NEXT: zip1 v0.2s, v0.2s, v5.2s
-; CHECK-IAENABLED-NEXT: fmul v5.2s, v1.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fmul v6.2s, v16.2s, v7.2s
-; CHECK-IAENABLED-NEXT: zip1 v7.2s, v2.2s, v4.2s
-; CHECK-IAENABLED-NEXT: zip2 v4.2s, v2.2s, v4.2s
-; CHECK-IAENABLED-NEXT: fmla v5.2s, v0.2s, v16.2s
-; CHECK-IAENABLED-NEXT: fneg v6.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmla v6.2s, v0.2s, v1.2s
-; CHECK-IAENABLED-NEXT: fmul v17.2s, v7.2s, v5.2s
+; CHECK-IAENABLED-NEXT: zip2 v1.2s, v1.2s, v4.2s
+; CHECK-IAENABLED-NEXT: movi v5.2s, #128, lsl #24
+; CHECK-IAENABLED-NEXT: fmul v4.2s, v6.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmls v5.2s, v1.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmla v4.2s, v0.2s, v1.2s
+; CHECK-IAENABLED-NEXT: zip1 v1.2s, v2.2s, v16.2s
+; CHECK-IAENABLED-NEXT: fmla v5.2s, v0.2s, v6.2s
+; CHECK-IAENABLED-NEXT: zip2 v6.2s, v2.2s, v16.2s
; CHECK-IAENABLED-NEXT: movi v0.2d, #0000000000000000
-; CHECK-IAENABLED-NEXT: fmul v1.2s, v4.2s, v5.2s
-; CHECK-IAENABLED-NEXT: fmla v17.2s, v6.2s, v4.2s
+; CHECK-IAENABLED-NEXT: fmul v18.2s, v1.2s, v4.2s
; CHECK-IAENABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
-; CHECK-IAENABLED-NEXT: str d6, [x0]
-; CHECK-IAENABLED-NEXT: fneg v16.2s, v1.2s
+; CHECK-IAENABLED-NEXT: str d5, [x0]
+; CHECK-IAENABLED-NEXT: fmla v18.2s, v5.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmls v17.2s, v6.2s, v4.2s
; CHECK-IAENABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
-; CHECK-IAENABLED-NEXT: fmla v16.2s, v6.2s, v7.2s
-; CHECK-IAENABLED-NEXT: st2 { v16.2s, v17.2s }, [x1]
+; CHECK-IAENABLED-NEXT: fmla v17.2s, v5.2s, v1.2s
+; CHECK-IAENABLED-NEXT: st2 { v17.2s, v18.2s }, [x1]
; CHECK-IAENABLED-NEXT: ret
;
; CHECK-IADISABLED-LABEL: multiple_muls_shuffle_external_intrinsic:
; CHECK-IADISABLED: // %bb.0: // %entry
; CHECK-IADISABLED-NEXT: mov d4, v1.d[1]
; CHECK-IADISABLED-NEXT: mov d5, v0.d[1]
+; CHECK-IADISABLED-NEXT: movi v16.2s, #128, lsl #24
; CHECK-IADISABLED-NEXT: zip1 v6.2s, v1.2s, v4.2s
; CHECK-IADISABLED-NEXT: zip2 v7.2s, v0.2s, v5.2s
; CHECK-IADISABLED-NEXT: zip1 v0.2s, v0.2s, v5.2s
; CHECK-IADISABLED-NEXT: zip2 v1.2s, v1.2s, v4.2s
; CHECK-IADISABLED-NEXT: mov d5, v2.d[1]
; CHECK-IADISABLED-NEXT: fmul v4.2s, v6.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fmla v4.2s, v0.2s, v1.2s
-; CHECK-IADISABLED-NEXT: fmul v1.2s, v1.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fmls v16.2s, v1.2s, v7.2s
; CHECK-IADISABLED-NEXT: zip2 v7.2s, v2.2s, v5.2s
; CHECK-IADISABLED-NEXT: zip1 v5.2s, v2.2s, v5.2s
-; CHECK-IADISABLED-NEXT: fneg v1.2s, v1.2s
-; CHECK-IADISABLED-NEXT: fmul v16.2s, v7.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fmul v4.2s, v5.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v6.2s
-; CHECK-IADISABLED-NEXT: fneg v6.2s, v16.2s
+; CHECK-IADISABLED-NEXT: fmla v4.2s, v0.2s, v1.2s
+; CHECK-IADISABLED-NEXT: movi v1.2s, #128, lsl #24
+; CHECK-IADISABLED-NEXT: fmla v16.2s, v0.2s, v6.2s
; CHECK-IADISABLED-NEXT: movi v0.2d, #0000000000000000
-; CHECK-IADISABLED-NEXT: fmla v4.2s, v1.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fmla v6.2s, v1.2s, v5.2s
+; CHECK-IADISABLED-NEXT: fmul v6.2s, v5.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmls v1.2s, v7.2s, v4.2s
; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #0
-; CHECK-IADISABLED-NEXT: str d1, [x0]
-; CHECK-IADISABLED-NEXT: mov v6.d[1], v4.d[0]
+; CHECK-IADISABLED-NEXT: str d16, [x0]
+; CHECK-IADISABLED-NEXT: fmla v6.2s, v16.2s, v7.2s
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v16.2s, v5.2s
; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v3.4s, #90
-; CHECK-IADISABLED-NEXT: rev64 v4.4s, v6.4s
-; CHECK-IADISABLED-NEXT: uzp1 v2.4s, v6.4s, v4.4s
-; CHECK-IADISABLED-NEXT: str q2, [x1]
+; CHECK-IADISABLED-NEXT: mov v1.d[1], v6.d[0]
+; CHECK-IADISABLED-NEXT: rev64 v4.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp1 v1.4s, v1.4s, v4.4s
+; CHECK-IADISABLED-NEXT: str q1, [x1]
; CHECK-IADISABLED-NEXT: ret
entry:
%a.deinterleaved = call { <2 x float>, <2 x float> } @llvm.vector.deinterleave2.v4f32(<4 x float> %a)
@@ -670,24 +670,24 @@ define <4 x float> @multiple_muls_shuffle_external_with_loads_intrinsic(ptr %ptr
; CHECK-IAENABLED-LABEL: multiple_muls_shuffle_external_with_loads_intrinsic:
; CHECK-IAENABLED: // %bb.0: // %entry
; CHECK-IAENABLED-NEXT: ld2 { v0.2s, v1.2s }, [x0]
+; CHECK-IAENABLED-NEXT: movi v4.2s, #128, lsl #24
; CHECK-IAENABLED-NEXT: ld2 { v2.2s, v3.2s }, [x1]
-; CHECK-IAENABLED-NEXT: fmul v4.2s, v3.2s, v1.2s
+; CHECK-IAENABLED-NEXT: fmls v4.2s, v3.2s, v1.2s
; CHECK-IAENABLED-NEXT: fmul v6.2s, v2.2s, v1.2s
-; CHECK-IAENABLED-NEXT: fneg v4.2s, v4.2s
-; CHECK-IAENABLED-NEXT: fmla v6.2s, v0.2s, v3.2s
; CHECK-IAENABLED-NEXT: fmla v4.2s, v0.2s, v2.2s
+; CHECK-IAENABLED-NEXT: fmla v6.2s, v0.2s, v3.2s
+; CHECK-IAENABLED-NEXT: movi v1.2s, #128, lsl #24
; CHECK-IAENABLED-NEXT: str d4, [x4]
; CHECK-IAENABLED-NEXT: ldr q5, [x2]
; CHECK-IAENABLED-NEXT: mov d7, v5.d[1]
; CHECK-IAENABLED-NEXT: zip1 v0.2s, v5.2s, v7.2s
-; CHECK-IAENABLED-NEXT: zip2 v1.2s, v5.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fmul v3.2s, v0.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmul v6.2s, v1.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmla v3.2s, v4.2s, v1.2s
-; CHECK-IAENABLED-NEXT: fneg v2.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmla v2.2s, v4.2s, v0.2s
+; CHECK-IAENABLED-NEXT: zip2 v3.2s, v5.2s, v7.2s
+; CHECK-IAENABLED-NEXT: fmul v2.2s, v0.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmla v2.2s, v4.2s, v3.2s
+; CHECK-IAENABLED-NEXT: fmls v1.2s, v3.2s, v6.2s
+; CHECK-IAENABLED-NEXT: fmla v1.2s, v4.2s, v0.2s
; CHECK-IAENABLED-NEXT: movi v0.2d, #0000000000000000
-; CHECK-IAENABLED-NEXT: st2 { v2.2s, v3.2s }, [x5]
+; CHECK-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x5]
; CHECK-IAENABLED-NEXT: ldr q1, [x3]
; CHECK-IAENABLED-NEXT: fcmla v0.4s, v5.4s, v1.4s, #0
; CHECK-IAENABLED-NEXT: fcmla v0.4s, v5.4s, v1.4s, #90
@@ -697,35 +697,35 @@ define <4 x float> @multiple_muls_shuffle_external_with_loads_intrinsic(ptr %ptr
; CHECK-IADISABLED: // %bb.0: // %entry
; CHECK-IADISABLED-NEXT: ldr q0, [x0]
; CHECK-IADISABLED-NEXT: ldr q1, [x1]
+; CHECK-IADISABLED-NEXT: movi v6.2s, #128, lsl #24
; CHECK-IADISABLED-NEXT: mov d2, v0.d[1]
; CHECK-IADISABLED-NEXT: mov d3, v1.d[1]
; CHECK-IADISABLED-NEXT: zip2 v4.2s, v0.2s, v2.2s
; CHECK-IADISABLED-NEXT: zip2 v5.2s, v1.2s, v3.2s
; CHECK-IADISABLED-NEXT: zip1 v0.2s, v0.2s, v2.2s
; CHECK-IADISABLED-NEXT: zip1 v1.2s, v1.2s, v3.2s
-; CHECK-IADISABLED-NEXT: fmul v6.2s, v5.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fneg v2.2s, v6.2s
-; CHECK-IADISABLED-NEXT: fmla v2.2s, v0.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmls v6.2s, v5.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmla v6.2s, v0.2s, v1.2s
; CHECK-IADISABLED-NEXT: fmul v1.2s, v1.2s, v4.2s
-; CHECK-IADISABLED-NEXT: str d2, [x4]
+; CHECK-IADISABLED-NEXT: str d6, [x4]
; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v5.2s
-; CHECK-IADISABLED-NEXT: ldr q3, [x2]
-; CHECK-IADISABLED-NEXT: mov d4, v3.d[1]
-; CHECK-IADISABLED-NEXT: zip2 v0.2s, v3.2s, v4.2s
-; CHECK-IADISABLED-NEXT: zip1 v4.2s, v3.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fmul v5.2s, v0.2s, v1.2s
-; CHECK-IADISABLED-NEXT: fmul v1.2s, v4.2s, v1.2s
-; CHECK-IADISABLED-NEXT: fneg v5.2s, v5.2s
-; CHECK-IADISABLED-NEXT: fmla v1.2s, v2.2s, v0.2s
-; CHECK-IADISABLED-NEXT: fmla v5.2s, v2.2s, v4.2s
-; CHECK-IADISABLED-NEXT: mov v5.d[1], v1.d[0]
-; CHECK-IADISABLED-NEXT: rev64 v0.4s, v5.4s
-; CHECK-IADISABLED-NEXT: uzp1 v1.4s, v5.4s, v0.4s
+; CHECK-IADISABLED-NEXT: movi v0.2s, #128, lsl #24
+; CHECK-IADISABLED-NEXT: ldr q2, [x2]
+; CHECK-IADISABLED-NEXT: mov d3, v2.d[1]
+; CHECK-IADISABLED-NEXT: zip2 v4.2s, v2.2s, v3.2s
+; CHECK-IADISABLED-NEXT: zip1 v3.2s, v2.2s, v3.2s
+; CHECK-IADISABLED-NEXT: fmul v5.2s, v3.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmls v0.2s, v4.2s, v1.2s
+; CHECK-IADISABLED-NEXT: fmla v5.2s, v6.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmla v0.2s, v6.2s, v3.2s
+; CHECK-IADISABLED-NEXT: mov v0.d[1], v5.d[0]
+; CHECK-IADISABLED-NEXT: rev64 v1.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp1 v1.4s, v0.4s, v1.4s
; CHECK-IADISABLED-NEXT: movi v0.2d, #0000000000000000
; CHECK-IADISABLED-NEXT: str q1, [x5]
; CHECK-IADISABLED-NEXT: ldr q1, [x3]
-; CHECK-IADISABLED-NEXT: fcmla v0.4s, v3.4s, v1.4s, #0
-; CHECK-IADISABLED-NEXT: fcmla v0.4s, v3.4s, v1.4s, #90
+; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v1.4s, #0
+; CHECK-IADISABLED-NEXT: fcmla v0.4s, v2.4s, v1.4s, #90
; CHECK-IADISABLED-NEXT: ret
entry:
%a = load <4 x float>, ptr %ptr_a
@@ -775,36 +775,36 @@ define <4 x float> @multiple_muls_mul_external_intrinsic(<4 x float> %a, <4 x fl
; CHECK-IAENABLED: // %bb.0: // %entry
; CHECK-IAENABLED-NEXT: mov d4, v0.d[1]
; CHECK-IAENABLED-NEXT: mov d5, v1.d[1]
-; CHECK-IAENABLED-NEXT: mov d16, v2.d[1]
-; CHECK-IAENABLED-NEXT: mov d17, v3.d[1]
-; CHECK-IAENABLED-NEXT: zip2 v6.2s, v0.2s, v4.2s
-; CHECK-IAENABLED-NEXT: zip2 v7.2s, v1.2s, v5.2s
-; CHECK-IAENABLED-NEXT: zip2 v19.2s, v2.2s, v16.2s
-; CHECK-IAENABLED-NEXT: zip1 v2.2s, v2.2s, v16.2s
-; CHECK-IAENABLED-NEXT: zip2 v16.2s, v3.2s, v17.2s
+; CHECK-IAENABLED-NEXT: mov d6, v2.d[1]
+; CHECK-IAENABLED-NEXT: mov d7, v3.d[1]
+; CHECK-IAENABLED-NEXT: movi v16.2s, #128, lsl #24
+; CHECK-IAENABLED-NEXT: zip2 v17.2s, v0.2s, v4.2s
+; CHECK-IAENABLED-NEXT: zip2 v18.2s, v1.2s, v5.2s
+; CHECK-IAENABLED-NEXT: zip1 v19.2s, v2.2s, v6.2s
+; CHECK-IAENABLED-NEXT: zip2 v20.2s, v3.2s, v7.2s
; CHECK-IAENABLED-NEXT: zip1 v0.2s, v0.2s, v4.2s
; CHECK-IAENABLED-NEXT: zip1 v1.2s, v1.2s, v5.2s
-; CHECK-IAENABLED-NEXT: zip1 v3.2s, v3.2s, v17.2s
-; CHECK-IAENABLED-NEXT: fmul v18.2s, v6.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fmul v5.2s, v2.2s, v16.2s
-; CHECK-IAENABLED-NEXT: fmul v16.2s, v19.2s, v16.2s
-; CHECK-IAENABLED-NEXT: fmul v7.2s, v0.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fneg v4.2s, v18.2s
-; CHECK-IAENABLED-NEXT: fmla v5.2s, v3.2s, v19.2s
-; CHECK-IAENABLED-NEXT: fmla v7.2s, v1.2s, v6.2s
-; CHECK-IAENABLED-NEXT: fmla v4.2s, v1.2s, v0.2s
-; CHECK-IAENABLED-NEXT: fneg v0.2s, v16.2s
-; CHECK-IAENABLED-NEXT: fmla v0.2s, v3.2s, v2.2s
-; CHECK-IAENABLED-NEXT: fmul v2.2s, v4.2s, v5.2s
-; CHECK-IAENABLED-NEXT: fmul v3.2s, v7.2s, v5.2s
-; CHECK-IAENABLED-NEXT: str d4, [x0]
-; CHECK-IAENABLED-NEXT: fmla v2.2s, v0.2s, v7.2s
-; CHECK-IAENABLED-NEXT: fneg v1.2s, v3.2s
-; CHECK-IAENABLED-NEXT: fmla v1.2s, v0.2s, v4.2s
-; CHECK-IAENABLED-NEXT: mov v0.d[1], v5.d[0]
-; CHECK-IAENABLED-NEXT: rev64 v3.4s, v0.4s
-; CHECK-IAENABLED-NEXT: st2 { v1.2s, v2.2s }, [x1]
-; CHECK-IAENABLED-NEXT: uzp1 v0.4s, v0.4s, v3.4s
+; CHECK-IAENABLED-NEXT: zip2 v2.2s, v2.2s, v6.2s
+; CHECK-IAENABLED-NEXT: movi v4.2s, #128, lsl #24
+; CHECK-IAENABLED-NEXT: zip1 v3.2s, v3.2s, v7.2s
+; CHECK-IAENABLED-NEXT: movi v6.2s, #128, lsl #24
+; CHECK-IAENABLED-NEXT: fmls v16.2s, v17.2s, v18.2s
+; CHECK-IAENABLED-NEXT: fmul v5.2s, v19.2s, v20.2s
+; CHECK-IAENABLED-NEXT: fmls v4.2s, v2.2s, v20.2s
+; CHECK-IAENABLED-NEXT: fmla v16.2s, v1.2s, v0.2s
+; CHECK-IAENABLED-NEXT: fmul v0.2s, v0.2s, v18.2s
+; CHECK-IAENABLED-NEXT: fmla v5.2s, v3.2s, v2.2s
+; CHECK-IAENABLED-NEXT: fmla v4.2s, v3.2s, v19.2s
+; CHECK-IAENABLED-NEXT: fmla v0.2s, v1.2s, v17.2s
+; CHECK-IAENABLED-NEXT: str d16, [x0]
+; CHECK-IAENABLED-NEXT: fmul v7.2s, v16.2s, v5.2s
+; CHECK-IAENABLED-NEXT: fmla v7.2s, v4.2s, v0.2s
+; CHECK-IAENABLED-NEXT: fmls v6.2s, v0.2s, v5.2s
+; CHECK-IAENABLED-NEXT: fmla v6.2s, v4.2s, v16.2s
+; CHECK-IAENABLED-NEXT: mov v4.d[1], v5.d[0]
+; CHECK-IAENABLED-NEXT: rev64 v0.4s, v4.4s
+; CHECK-IAENABLED-NEXT: st2 { v6.2s, v7.2s }, [x1]
+; CHECK-IAENABLED-NEXT: uzp1 v0.4s, v4.4s, v0.4s
; CHECK-IAENABLED-NEXT: ret
;
; CHECK-IADISABLED-LABEL: multiple_muls_mul_external_intrinsic:
@@ -813,36 +813,36 @@ define <4 x float> @multiple_muls_mul_external_intrinsic(<4 x float> %a, <4 x fl
; CHECK-IADISABLED-NEXT: mov d5, v1.d[1]
; CHECK-IADISABLED-NEXT: mov d6, v2.d[1]
; CHECK-IADISABLED-NEXT: mov d7, v3.d[1]
-; CHECK-IADISABLED-NEXT: zip1 v16.2s, v0.2s, v4.2s
-; CHECK-IADISABLED-NEXT: zip2 v17.2s, v1.2s, v5.2s
-; CHECK-IADISABLED-NEXT: zip2 v0.2s, v0.2s, v4.2s
-; CHECK-IADISABLED-NEXT: zip1 v4.2s, v2.2s, v6.2s
-; CHECK-IADISABLED-NEXT: zip2 v18.2s, v3.2s, v7.2s
+; CHECK-IADISABLED-NEXT: movi v16.2s, #128, lsl #24
+; CHECK-IADISABLED-NEXT: zip2 v17.2s, v0.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip1 v0.2s, v0.2s, v4.2s
+; CHECK-IADISABLED-NEXT: zip2 v4.2s, v1.2s, v5.2s
+; CHECK-IADISABLED-NEXT: zip1 v18.2s, v2.2s, v6.2s
+; CHECK-IADISABLED-NEXT: zip2 v19.2s, v3.2s, v7.2s
; CHECK-IADISABLED-NEXT: zip1 v1.2s, v1.2s, v5.2s
; CHECK-IADISABLED-NEXT: zip2 v2.2s, v2.2s, v6.2s
; CHECK-IADISABLED-NEXT: zip1 v3.2s, v3.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fmul v5.2s, v16.2s, v17.2s
-; CHECK-IADISABLED-NEXT: fmul v6.2s, v0.2s, v17.2s
-; CHECK-IADISABLED-NEXT: fmul v7.2s, v4.2s, v18.2s
-; CHECK-IADISABLED-NEXT: fmla v5.2s, v1.2s, v0.2s
-; CHECK-IADISABLED-NEXT: fmul v0.2s, v2.2s, v18.2s
-; CHECK-IADISABLED-NEXT: fmla v7.2s, v3.2s, v2.2s
-; CHECK-IADISABLED-NEXT: fneg v2.2s, v6.2s
-; CHECK-IADISABLED-NEXT: fneg v0.2s, v0.2s
-; CHECK-IADISABLED-NEXT: fmla v2.2s, v1.2s, v16.2s
-; CHECK-IADISABLED-NEXT: fmul v1.2s, v5.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fmla v0.2s, v3.2s, v4.2s
-; CHECK-IADISABLED-NEXT: fmul v3.2s, v2.2s, v7.2s
-; CHECK-IADISABLED-NEXT: fneg v1.2s, v1.2s
-; CHECK-IADISABLED-NEXT: str d2, [x0]
-; CHECK-IADISABLED-NEXT: fmla v3.2s, v0.2s, v5.2s
-; CHECK-IADISABLED-NEXT: fmla v1.2s, v0.2s, v2.2s
-; CHECK-IADISABLED-NEXT: mov v0.d[1], v7.d[0]
-; CHECK-IADISABLED-NEXT: mov v1.d[1], v3.d[0]
-; CHECK-IADISABLED-NEXT: rev64 v3.4s, v0.4s
-; CHECK-IADISABLED-NEXT: rev64 v4.4s, v1.4s
-; CHECK-IADISABLED-NEXT: uzp1 v0.4s, v0.4s, v3.4s
-; CHECK-IADISABLED-NEXT: uzp1 v1.4s, v1.4s, v4.4s
+; CHECK-IADISABLED-NEXT: fmul v5.2s, v0.2s, v4.2s
+; CHECK-IADISABLED-NEXT: fmls v16.2s, v17.2s, v4.2s
+; CHECK-IADISABLED-NEXT: movi v4.2s, #128, lsl #24
+; CHECK-IADISABLED-NEXT: fmul v6.2s, v18.2s, v19.2s
+; CHECK-IADISABLED-NEXT: fmla v5.2s, v1.2s, v17.2s
+; CHECK-IADISABLED-NEXT: fmla v16.2s, v1.2s, v0.2s
+; CHECK-IADISABLED-NEXT: movi v1.2s, #128, lsl #24
+; CHECK-IADISABLED-NEXT: fmla v6.2s, v3.2s, v2.2s
+; CHECK-IADISABLED-NEXT: fmls v4.2s, v2.2s, v19.2s
+; CHECK-IADISABLED-NEXT: str d16, [x0]
+; CHECK-IADISABLED-NEXT: fmla v4.2s, v3.2s, v18.2s
+; CHECK-IADISABLED-NEXT: fmul v0.2s, v16.2s, v6.2s
+; CHECK-IADISABLED-NEXT: fmls v1.2s, v5.2s, v6.2s
+; CHECK-IADISABLED-NEXT: fmla v0.2s, v4.2s, v5.2s
+; CHECK-IADISABLED-NEXT: fmla v1.2s, v4.2s, v16.2s
+; CHECK-IADISABLED-NEXT: mov v4.d[1], v6.d[0]
+; CHECK-IADISABLED-NEXT: mov v1.d[1], v0.d[0]
+; CHECK-IADISABLED-NEXT: rev64 v0.4s, v4.4s
+; CHECK-IADISABLED-NEXT: rev64 v2.4s, v1.4s
+; CHECK-IADISABLED-NEXT: uzp1 v0.4s, v4.4s, v0.4s
+; CHECK-IADISABLED-NEXT: uzp1 v1.4s, v1.4s, v2.4s
; CHECK-IADISABLED-NEXT: str q1, [x1]
; CHECK-IADISABLED-NEXT: ret
entry:
diff --git a/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll b/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
index 6eb7cf35c2a42..5bee01cf9e607 100644
--- a/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
+++ b/llvm/test/CodeGen/AArch64/complex-deinterleaving-uniform-cases.ll
@@ -201,95 +201,91 @@ entry:
define <12 x float> @abp90c12(<12 x float> %a, <12 x float> %b, <12 x float> %c) {
; CHECK-LABEL: abp90c12:
; CHECK: // %bb.0: // %entry
-; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
-; CHECK-NEXT: // kill: def $s2 killed $s2 def $q2
; CHECK-NEXT: // kill: def $s3 killed $s3 def $q3
-; CHECK-NEXT: ldr s17, [sp, #32]
-; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4
+; CHECK-NEXT: // kill: def $s2 killed $s2 def $q2
+; CHECK-NEXT: // kill: def $s0 killed $s0 def $q0
; CHECK-NEXT: add x10, sp, #48
-; CHECK-NEXT: add x11, sp, #64
-; CHECK-NEXT: mov v0.s[1], v2.s[0]
-; CHECK-NEXT: mov v1.s[1], v3.s[0]
+; CHECK-NEXT: ldr s16, [sp, #40]
; CHECK-NEXT: // kill: def $s5 killed $s5 def $q5
+; CHECK-NEXT: add x9, sp, #16
+; CHECK-NEXT: mov v1.s[1], v3.s[0]
+; CHECK-NEXT: mov v0.s[1], v2.s[0]
+; CHECK-NEXT: ldr s2, [sp, #32]
+; CHECK-NEXT: ldr s3, [sp]
+; CHECK-NEXT: // kill: def $s4 killed $s4 def $q4
+; CHECK-NEXT: ldr s17, [sp, #8]
+; CHECK-NEXT: // kill: def $s7 killed $s7 def $q7
; CHECK-NEXT: // kill: def $s6 killed $s6 def $q6
-; CHECK-NEXT: ldr s3, [sp, #96]
-; CHECK-NEXT: ld1 { v17.s }[1], [x10]
-; CHECK-NEXT: add x10, sp, #72
-; CHECK-NEXT: ldr s16, [sp, #8]
-; CHECK-NEXT: ldr s18, [x10]
+; CHECK-NEXT: movi v18.4s, #128, lsl #24
+; CHECK-NEXT: ld1 { v2.s }[1], [x10]
; CHECK-NEXT: add x10, sp, #56
-; CHECK-NEXT: // kill: def $s7 killed $s7 def $q7
-; CHECK-NEXT: ldr s2, [sp]
-; CHECK-NEXT: add x9, sp, #16
-; CHECK-NEXT: mov v0.s[2], v4.s[0]
-; CHECK-NEXT: ldr s4, [x11]
+; CHECK-NEXT: ldr s21, [sp, #136]
+; CHECK-NEXT: ld1 { v16.s }[1], [x10]
+; CHECK-NEXT: ld1 { v3.s }[1], [x9]
+; CHECK-NEXT: add x9, sp, #64
; CHECK-NEXT: mov v1.s[2], v5.s[0]
-; CHECK-NEXT: add x11, sp, #80
-; CHECK-NEXT: ldr s5, [sp, #40]
-; CHECK-NEXT: ld1 { v2.s }[1], [x9]
-; CHECK-NEXT: ld1 { v4.s }[1], [x11]
-; CHECK-NEXT: add x11, sp, #88
-; CHECK-NEXT: add x9, sp, #168
-; CHECK-NEXT: ld1 { v5.s }[1], [x10]
-; CHECK-NEXT: ld1 { v18.s }[1], [x11]
-; CHECK-NEXT: add x11, sp, #112
-; CHECK-NEXT: mov v0.s[3], v6.s[0]
-; CHECK-NEXT: mov v1.s[3], v7.s[0]
+; CHECK-NEXT: mov v0.s[2], v4.s[0]
; CHECK-NEXT: add x10, sp, #24
-; CHECK-NEXT: ld1 { v3.s }[1], [x11]
-; CHECK-NEXT: zip1 v4.2d, v17.2d, v4.2d
-; CHECK-NEXT: add x11, sp, #120
-; CHECK-NEXT: zip1 v6.2d, v5.2d, v18.2d
-; CHECK-NEXT: ldr s5, [sp, #104]
-; CHECK-NEXT: ld1 { v16.s }[1], [x10]
-; CHECK-NEXT: add x10, sp, #160
-; CHECK-NEXT: ldr s7, [sp, #128]
-; CHECK-NEXT: ldr s18, [sp, #192]
-; CHECK-NEXT: ld1 { v5.s }[1], [x11]
-; CHECK-NEXT: ldr s17, [x10]
-; CHECK-NEXT: add x10, sp, #144
-; CHECK-NEXT: add x11, sp, #176
-; CHECK-NEXT: ld1 { v7.s }[1], [x10]
-; CHECK-NEXT: ldr s21, [x9]
-; CHECK-NEXT: ld1 { v17.s }[1], [x11]
-; CHECK-NEXT: fmul v19.4s, v6.4s, v1.4s
-; CHECK-NEXT: fmul v1.4s, v4.4s, v1.4s
-; CHECK-NEXT: fmul v20.4s, v5.4s, v16.4s
-; CHECK-NEXT: fmul v16.4s, v3.4s, v16.4s
-; CHECK-NEXT: add x9, sp, #208
-; CHECK-NEXT: add x10, sp, #152
-; CHECK-NEXT: add x11, sp, #184
-; CHECK-NEXT: ld1 { v18.s }[1], [x9]
-; CHECK-NEXT: zip1 v7.2d, v7.2d, v17.2d
-; CHECK-NEXT: ldr s17, [sp, #136]
-; CHECK-NEXT: ld1 { v21.s }[1], [x11]
-; CHECK-NEXT: fneg v19.4s, v19.4s
-; CHECK-NEXT: fmla v1.4s, v0.4s, v6.4s
-; CHECK-NEXT: add x9, sp, #216
-; CHECK-NEXT: fneg v20.4s, v20.4s
-; CHECK-NEXT: fmla v16.4s, v2.4s, v5.4s
+; CHECK-NEXT: ld1 { v2.s }[2], [x9]
+; CHECK-NEXT: add x9, sp, #72
+; CHECK-NEXT: ldr s4, [sp, #96]
+; CHECK-NEXT: ld1 { v16.s }[2], [x9]
; CHECK-NEXT: ld1 { v17.s }[1], [x10]
-; CHECK-NEXT: ldr s5, [sp, #200]
-; CHECK-NEXT: zip1 v6.2d, v17.2d, v21.2d
-; CHECK-NEXT: fmla v19.4s, v0.4s, v4.4s
-; CHECK-NEXT: fsub v0.4s, v7.4s, v1.4s
-; CHECK-NEXT: fmla v20.4s, v2.4s, v3.4s
-; CHECK-NEXT: fsub v1.4s, v18.4s, v16.4s
+; CHECK-NEXT: add x10, sp, #112
+; CHECK-NEXT: add x9, sp, #80
+; CHECK-NEXT: ld1 { v4.s }[1], [x10]
+; CHECK-NEXT: ldr s5, [sp, #128]
+; CHECK-NEXT: mov v1.s[3], v7.s[0]
+; CHECK-NEXT: ld1 { v2.s }[3], [x9]
+; CHECK-NEXT: add x9, sp, #88
+; CHECK-NEXT: ld1 { v16.s }[3], [x9]
+; CHECK-NEXT: add x9, sp, #144
+; CHECK-NEXT: mov v0.s[3], v6.s[0]
+; CHECK-NEXT: movi v6.4s, #128, lsl #24
+; CHECK-NEXT: ldr s7, [sp, #104]
; CHECK-NEXT: ld1 { v5.s }[1], [x9]
-; CHECK-NEXT: fadd v2.4s, v6.4s, v19.4s
-; CHECK-NEXT: fadd v3.4s, v5.4s, v20.4s
+; CHECK-NEXT: add x10, sp, #120
+; CHECK-NEXT: fmul v19.4s, v4.4s, v17.4s
+; CHECK-NEXT: add x9, sp, #160
+; CHECK-NEXT: fmul v20.4s, v2.4s, v1.4s
+; CHECK-NEXT: ld1 { v7.s }[1], [x10]
+; CHECK-NEXT: add x10, sp, #208
+; CHECK-NEXT: ld1 { v5.s }[2], [x9]
+; CHECK-NEXT: add x9, sp, #152
+; CHECK-NEXT: fmls v6.4s, v16.4s, v1.4s
+; CHECK-NEXT: ldr s1, [sp, #192]
+; CHECK-NEXT: ld1 { v21.s }[1], [x9]
+; CHECK-NEXT: fmls v18.4s, v7.4s, v17.4s
+; CHECK-NEXT: add x9, sp, #176
+; CHECK-NEXT: fmla v19.4s, v3.4s, v7.4s
+; CHECK-NEXT: fmla v20.4s, v0.4s, v16.4s
+; CHECK-NEXT: ld1 { v1.s }[1], [x10]
+; CHECK-NEXT: ld1 { v5.s }[3], [x9]
+; CHECK-NEXT: add x9, sp, #168
+; CHECK-NEXT: ldr s7, [sp, #200]
+; CHECK-NEXT: add x10, sp, #184
+; CHECK-NEXT: ld1 { v21.s }[2], [x9]
+; CHECK-NEXT: add x9, sp, #216
+; CHECK-NEXT: fmla v6.4s, v0.4s, v2.4s
+; CHECK-NEXT: fmla v18.4s, v3.4s, v4.4s
+; CHECK-NEXT: fsub v1.4s, v1.4s, v19.4s
+; CHECK-NEXT: ld1 { v7.s }[1], [x9]
+; CHECK-NEXT: fsub v0.4s, v5.4s, v20.4s
+; CHECK-NEXT: ld1 { v21.s }[3], [x10]
+; CHECK-NEXT: fadd v2.4s, v7.4s, v18.4s
+; CHECK-NEXT: fadd v3.4s, v21.4s, v6.4s
; CHECK-NEXT: ext v4.16b, v0.16b, v1.16b, #12
-; CHECK-NEXT: ext v5.16b, v2.16b, v3.16b, #12
+; CHECK-NEXT: trn2 v1.4s, v1.4s, v2.4s
+; CHECK-NEXT: ext v5.16b, v3.16b, v2.16b, #12
; CHECK-NEXT: ext v4.16b, v0.16b, v4.16b, #12
-; CHECK-NEXT: trn2 v1.4s, v1.4s, v3.4s
-; CHECK-NEXT: ext v5.16b, v2.16b, v5.16b, #8
+; CHECK-NEXT: ext v5.16b, v3.16b, v5.16b, #8
; CHECK-NEXT: rev64 v4.4s, v4.4s
-; CHECK-NEXT: trn2 v3.4s, v4.4s, v5.4s
-; CHECK-NEXT: zip2 v4.4s, v0.4s, v2.4s
-; CHECK-NEXT: zip1 v0.4s, v0.4s, v2.4s
-; CHECK-NEXT: ext v1.16b, v3.16b, v1.16b, #8
-; CHECK-NEXT: mov v4.d[1], v3.d[0]
+; CHECK-NEXT: trn2 v2.4s, v4.4s, v5.4s
+; CHECK-NEXT: zip2 v4.4s, v0.4s, v3.4s
+; CHECK-NEXT: zip1 v0.4s, v0.4s, v3.4s
+; CHECK-NEXT: ext v1.16b, v2.16b, v1.16b, #8
+; CHECK-NEXT: mov v4.d[1], v2.d[0]
; CHECK-NEXT: str q0, [x8]
; CHECK-NEXT: stp q4, q1, [x8, #16]
; CHECK-NEXT: ret
diff --git a/llvm/test/CodeGen/AArch64/neon-fnmul.ll b/llvm/test/CodeGen/AArch64/neon-fnmul.ll
new file mode 100644
index 0000000000000..01bf658912767
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/neon-fnmul.ll
@@ -0,0 +1,624 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=aarch64 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-NOFP16
+; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD,CHECK-SD-FP16
+; RUN: llc -mtriple=aarch64 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-NOFP16
+; RUN: llc -mtriple=aarch64 -mattr=+fullfp16 -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI,CHECK-GI-FP16
+
+; CHECK-GI: warning: Instruction selection used fallback path for fmul_neg_strict
+; CHECK-GI-NEXT: warning: Instruction selection used fallback path for neg_fmul_strict
+
+define <4 x float> @fmul_neg_v4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: fmul_neg_v4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.4s, #128, lsl #24
+; CHECK-NEXT: fmls v2.4s, v1.4s, v0.4s
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %n = fneg <4 x float> %b
+ %m = fmul <4 x float> %a, %n
+ ret <4 x float> %m
+}
+
+define <2 x float> @fmul_neg_v2f32(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: fmul_neg_v2f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.2s, #128, lsl #24
+; CHECK-NEXT: fmls v2.2s, v1.2s, v0.2s
+; CHECK-NEXT: fmov d0, d2
+; CHECK-NEXT: ret
+ %n = fneg <2 x float> %b
+ %m = fmul <2 x float> %a, %n
+ ret <2 x float> %m
+}
+
+define <8 x half> @fmul_neg_v8f16(<8 x half> %a, <8 x half> %b) {
+; CHECK-SD-NOFP16-LABEL: fmul_neg_v8f16:
+; CHECK-SD-NOFP16: // %bb.0:
+; CHECK-SD-NOFP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-SD-NOFP16-NEXT: fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT: eor v1.16b, v1.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT: fcvtl v2.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT: fmul v2.4s, v3.4s, v2.4s
+; CHECK-SD-NOFP16-NEXT: fmul v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT: fcvtn2 v0.8h, v1.4s
+; CHECK-SD-NOFP16-NEXT: ret
+;
+; CHECK-SD-FP16-LABEL: fmul_neg_v8f16:
+; CHECK-SD-FP16: // %bb.0:
+; CHECK-SD-FP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-SD-FP16-NEXT: fmls v2.8h, v1.8h, v0.8h
+; CHECK-SD-FP16-NEXT: mov v0.16b, v2.16b
+; CHECK-SD-FP16-NEXT: ret
+;
+; CHECK-GI-NOFP16-LABEL: fmul_neg_v8f16:
+; CHECK-GI-NOFP16: // %bb.0:
+; CHECK-GI-NOFP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT: eor v1.16b, v1.16b, v2.16b
+; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT: fmul v2.4s, v3.4s, v2.4s
+; CHECK-GI-NOFP16-NEXT: fmul v1.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v2.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn2 v0.8h, v1.4s
+; CHECK-GI-NOFP16-NEXT: ret
+;
+; CHECK-GI-FP16-LABEL: fmul_neg_v8f16:
+; CHECK-GI-FP16: // %bb.0:
+; CHECK-GI-FP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-GI-FP16-NEXT: fmls v2.8h, v1.8h, v0.8h
+; CHECK-GI-FP16-NEXT: mov v0.16b, v2.16b
+; CHECK-GI-FP16-NEXT: ret
+ %n = fneg <8 x half> %b
+ %m = fmul <8 x half> %a, %n
+ ret <8 x half> %m
+}
+
+define <4 x half> @fmul_neg_v4f16(<4 x half> %a, <4 x half> %b) {
+; CHECK-SD-NOFP16-LABEL: fmul_neg_v4f16:
+; CHECK-SD-NOFP16: // %bb.0:
+; CHECK-SD-NOFP16-NEXT: movi v2.4h, #128, lsl #8
+; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT: eor v1.8b, v1.8b, v2.8b
+; CHECK-SD-NOFP16-NEXT: fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT: ret
+;
+; CHECK-SD-FP16-LABEL: fmul_neg_v4f16:
+; CHECK-SD-FP16: // %bb.0:
+; CHECK-SD-FP16-NEXT: movi v2.4h, #128, lsl #8
+; CHECK-SD-FP16-NEXT: fmls v2.4h, v1.4h, v0.4h
+; CHECK-SD-FP16-NEXT: fmov d0, d2
+; CHECK-SD-FP16-NEXT: ret
+;
+; CHECK-GI-NOFP16-LABEL: fmul_neg_v4f16:
+; CHECK-GI-NOFP16: // %bb.0:
+; CHECK-GI-NOFP16-NEXT: movi v2.4h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT: eor v1.8b, v1.8b, v2.8b
+; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT: ret
+;
+; CHECK-GI-FP16-LABEL: fmul_neg_v4f16:
+; CHECK-GI-FP16: // %bb.0:
+; CHECK-GI-FP16-NEXT: movi v2.4h, #128, lsl #8
+; CHECK-GI-FP16-NEXT: fmls v2.4h, v1.4h, v0.4h
+; CHECK-GI-FP16-NEXT: fmov d0, d2
+; CHECK-GI-FP16-NEXT: ret
+ %n = fneg <4 x half> %b
+ %m = fmul <4 x half> %a, %n
+ ret <4 x half> %m
+}
+
+define <4 x float> @neg_fmul_v4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: neg_fmul_v4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.4s, #128, lsl #24
+; CHECK-NEXT: fmls v2.4s, v0.4s, v1.4s
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %m = fmul <4 x float> %a, %b
+ %n = fneg <4 x float> %m
+ ret <4 x float> %n
+}
+
+define <2 x float> @neg_fmul_v2f32(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: neg_fmul_v2f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.2s, #128, lsl #24
+; CHECK-NEXT: fmls v2.2s, v0.2s, v1.2s
+; CHECK-NEXT: fmov d0, d2
+; CHECK-NEXT: ret
+ %m = fmul <2 x float> %a, %b
+ %n = fneg <2 x float> %m
+ ret <2 x float> %n
+}
+
+define <8 x half> @neg_fmul_v8f16(<8 x half> %a, <8 x half> %b) {
+; CHECK-SD-NOFP16-LABEL: neg_fmul_v8f16:
+; CHECK-SD-NOFP16: // %bb.0:
+; CHECK-SD-NOFP16-NEXT: fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT: fcvtl v4.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-SD-NOFP16-NEXT: fmul v3.4s, v4.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT: fcvtn v1.4h, v3.4s
+; CHECK-SD-NOFP16-NEXT: fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT: eor v0.16b, v1.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT: ret
+;
+; CHECK-SD-FP16-LABEL: neg_fmul_v8f16:
+; CHECK-SD-FP16: // %bb.0:
+; CHECK-SD-FP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-SD-FP16-NEXT: fmls v2.8h, v0.8h, v1.8h
+; CHECK-SD-FP16-NEXT: mov v0.16b, v2.16b
+; CHECK-SD-FP16-NEXT: ret
+;
+; CHECK-GI-NOFP16-LABEL: neg_fmul_v8f16:
+; CHECK-GI-NOFP16: // %bb.0:
+; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT: fcvtl v4.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h
+; CHECK-GI-NOFP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT: fmul v3.4s, v3.4s, v4.4s
+; CHECK-GI-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v3.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn2 v1.8h, v0.4s
+; CHECK-GI-NOFP16-NEXT: eor v0.16b, v1.16b, v2.16b
+; CHECK-GI-NOFP16-NEXT: ret
+;
+; CHECK-GI-FP16-LABEL: neg_fmul_v8f16:
+; CHECK-GI-FP16: // %bb.0:
+; CHECK-GI-FP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-GI-FP16-NEXT: fmls v2.8h, v0.8h, v1.8h
+; CHECK-GI-FP16-NEXT: mov v0.16b, v2.16b
+; CHECK-GI-FP16-NEXT: ret
+ %m = fmul <8 x half> %a, %b
+ %n = fneg <8 x half> %m
+ ret <8 x half> %n
+}
+
+define <4 x half> @neg_fmul_v4f16(<4 x half> %a, <4 x half> %b) {
+; CHECK-SD-NOFP16-LABEL: neg_fmul_v4f16:
+; CHECK-SD-NOFP16: // %bb.0:
+; CHECK-SD-NOFP16-NEXT: fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT: movi v2.4h, #128, lsl #8
+; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT: eor v0.8b, v0.8b, v2.8b
+; CHECK-SD-NOFP16-NEXT: ret
+;
+; CHECK-SD-FP16-LABEL: neg_fmul_v4f16:
+; CHECK-SD-FP16: // %bb.0:
+; CHECK-SD-FP16-NEXT: movi v2.4h, #128, lsl #8
+; CHECK-SD-FP16-NEXT: fmls v2.4h, v0.4h, v1.4h
+; CHECK-SD-FP16-NEXT: fmov d0, d2
+; CHECK-SD-FP16-NEXT: ret
+;
+; CHECK-GI-NOFP16-LABEL: neg_fmul_v4f16:
+; CHECK-GI-NOFP16: // %bb.0:
+; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT: movi v2.4h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT: eor v0.8b, v0.8b, v2.8b
+; CHECK-GI-NOFP16-NEXT: ret
+;
+; CHECK-GI-FP16-LABEL: neg_fmul_v4f16:
+; CHECK-GI-FP16: // %bb.0:
+; CHECK-GI-FP16-NEXT: movi v2.4h, #128, lsl #8
+; CHECK-GI-FP16-NEXT: fmls v2.4h, v0.4h, v1.4h
+; CHECK-GI-FP16-NEXT: fmov d0, d2
+; CHECK-GI-FP16-NEXT: ret
+ %m = fmul <4 x half> %a, %b
+ %n = fneg <4 x half> %m
+ ret <4 x half> %n
+}
+
+define <4 x float> @fmul_neg_commuted_v4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: fmul_neg_commuted_v4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.4s, #128, lsl #24
+; CHECK-NEXT: fmls v2.4s, v1.4s, v0.4s
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %n = fneg <4 x float> %b
+ %m = fmul <4 x float> %n, %a
+ ret <4 x float> %m
+}
+
+define <4 x float> @neg_fmul_lane_v4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: neg_fmul_lane_v4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.4s, #128, lsl #24
+; CHECK-NEXT: fmls v2.4s, v0.4s, v1.s[1]
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %sp = shufflevector <4 x float> %b, <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+ %m = fmul <4 x float> %a, %sp
+ %n = fneg <4 x float> %m
+ ret <4 x float> %n
+}
+
+define <4 x float> @fmul_neg_lane_v4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: fmul_neg_lane_v4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.4s, #128, lsl #24
+; CHECK-NEXT: fmls v2.4s, v0.4s, v1.s[1]
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %sp = shufflevector <4 x float> %b, <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+ %n = fneg <4 x float> %sp
+ %m = fmul <4 x float> %a, %n
+ ret <4 x float> %m
+}
+
+define <4 x float> @neg_fmul_dup_v4f32(<4 x float> %a, float %b) {
+; CHECK-LABEL: neg_fmul_dup_v4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.4s, #128, lsl #24
+; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
+; CHECK-NEXT: fmls v2.4s, v0.4s, v1.s[0]
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %ie = insertelement <4 x float> poison, float %b, i32 0
+ %sp = shufflevector <4 x float> %ie, <4 x float> poison, <4 x i32> zeroinitializer
+ %m = fmul <4 x float> %a, %sp
+ %n = fneg <4 x float> %m
+ ret <4 x float> %n
+}
+
+define <2 x float> @neg_fmul_lane_v2f32(<2 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: neg_fmul_lane_v2f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.2s, #128, lsl #24
+; CHECK-NEXT: fmls v2.2s, v0.2s, v1.s[3]
+; CHECK-NEXT: fmov d0, d2
+; CHECK-NEXT: ret
+ %sp = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 3, i32 3>
+ %m = fmul <2 x float> %a, %sp
+ %n = fneg <2 x float> %m
+ ret <2 x float> %n
+}
+
+define <8 x half> @neg_fmul_lane_v8f16(<8 x half> %a, <8 x half> %b) {
+; CHECK-SD-NOFP16-LABEL: neg_fmul_lane_v8f16:
+; CHECK-SD-NOFP16: // %bb.0:
+; CHECK-SD-NOFP16-NEXT: dup v1.8h, v1.h[3]
+; CHECK-SD-NOFP16-NEXT: fcvtl v3.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-SD-NOFP16-NEXT: fcvtl v4.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT: fmul v3.4s, v3.4s, v4.4s
+; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT: fcvtn v1.4h, v3.4s
+; CHECK-SD-NOFP16-NEXT: fcvtn2 v1.8h, v0.4s
+; CHECK-SD-NOFP16-NEXT: eor v0.16b, v1.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT: ret
+;
+; CHECK-SD-FP16-LABEL: neg_fmul_lane_v8f16:
+; CHECK-SD-FP16: // %bb.0:
+; CHECK-SD-FP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-SD-FP16-NEXT: fmls v2.8h, v0.8h, v1.h[3]
+; CHECK-SD-FP16-NEXT: mov v0.16b, v2.16b
+; CHECK-SD-FP16-NEXT: ret
+;
+; CHECK-GI-NOFP16-LABEL: neg_fmul_lane_v8f16:
+; CHECK-GI-NOFP16: // %bb.0:
+; CHECK-GI-NOFP16-NEXT: dup v1.4h, v1.h[3]
+; CHECK-GI-NOFP16-NEXT: fcvtl v3.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT: fmul v3.4s, v3.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn v1.4h, v3.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn2 v1.8h, v0.4s
+; CHECK-GI-NOFP16-NEXT: eor v0.16b, v1.16b, v2.16b
+; CHECK-GI-NOFP16-NEXT: ret
+;
+; CHECK-GI-FP16-LABEL: neg_fmul_lane_v8f16:
+; CHECK-GI-FP16: // %bb.0:
+; CHECK-GI-FP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-GI-FP16-NEXT: fmls v2.8h, v0.8h, v1.h[3]
+; CHECK-GI-FP16-NEXT: mov v0.16b, v2.16b
+; CHECK-GI-FP16-NEXT: ret
+ %sp = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+ %m = fmul <8 x half> %a, %sp
+ %n = fneg <8 x half> %m
+ ret <8 x half> %n
+}
+
+define <4 x half> @neg_fmul_lane_v4f16(<4 x half> %a, <8 x half> %b) {
+; CHECK-SD-NOFP16-LABEL: neg_fmul_lane_v4f16:
+; CHECK-SD-NOFP16: // %bb.0:
+; CHECK-SD-NOFP16-NEXT: dup v1.4h, v1.h[5]
+; CHECK-SD-NOFP16-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT: movi v2.4h, #128, lsl #8
+; CHECK-SD-NOFP16-NEXT: fcvtl v1.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-SD-NOFP16-NEXT: eor v0.8b, v0.8b, v2.8b
+; CHECK-SD-NOFP16-NEXT: ret
+;
+; CHECK-SD-FP16-LABEL: neg_fmul_lane_v4f16:
+; CHECK-SD-FP16: // %bb.0:
+; CHECK-SD-FP16-NEXT: movi v2.4h, #128, lsl #8
+; CHECK-SD-FP16-NEXT: fmls v2.4h, v0.4h, v1.h[5]
+; CHECK-SD-FP16-NEXT: fmov d0, d2
+; CHECK-SD-FP16-NEXT: ret
+;
+; CHECK-GI-NOFP16-LABEL: neg_fmul_lane_v4f16:
+; CHECK-GI-NOFP16: // %bb.0:
+; CHECK-GI-NOFP16-NEXT: dup v1.4h, v1.h[5]
+; CHECK-GI-NOFP16-NEXT: fcvtl v0.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT: movi v2.4h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v0.4s
+; CHECK-GI-NOFP16-NEXT: eor v0.8b, v0.8b, v2.8b
+; CHECK-GI-NOFP16-NEXT: ret
+;
+; CHECK-GI-FP16-LABEL: neg_fmul_lane_v4f16:
+; CHECK-GI-FP16: // %bb.0:
+; CHECK-GI-FP16-NEXT: movi v2.4h, #128, lsl #8
+; CHECK-GI-FP16-NEXT: fmls v2.4h, v0.4h, v1.h[5]
+; CHECK-GI-FP16-NEXT: fmov d0, d2
+; CHECK-GI-FP16-NEXT: ret
+ %sp = shufflevector <8 x half> %b, <8 x half> poison, <4 x i32> <i32 5, i32 5, i32 5, i32 5>
+ %m = fmul <4 x half> %a, %sp
+ %n = fneg <4 x half> %m
+ ret <4 x half> %n
+}
+
+define <4 x float> @fneg_lhs_lane_v4f32(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: fneg_lhs_lane_v4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.4s, #128, lsl #24
+; CHECK-NEXT: fmls v2.4s, v0.4s, v1.s[1]
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %sp = shufflevector <4 x float> %b, <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+ %n = fneg <4 x float> %a
+ %m = fmul <4 x float> %n, %sp
+ ret <4 x float> %m
+}
+
+define <4 x float> @fneg_lhs_dup_v4f32(<4 x float> %a, float %b) {
+; CHECK-LABEL: fneg_lhs_dup_v4f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.4s, #128, lsl #24
+; CHECK-NEXT: // kill: def $s1 killed $s1 def $q1
+; CHECK-NEXT: fmls v2.4s, v0.4s, v1.s[0]
+; CHECK-NEXT: mov v0.16b, v2.16b
+; CHECK-NEXT: ret
+ %ie = insertelement <4 x float> poison, float %b, i32 0
+ %sp = shufflevector <4 x float> %ie, <4 x float> poison, <4 x i32> zeroinitializer
+ %n = fneg <4 x float> %a
+ %m = fmul <4 x float> %n, %sp
+ ret <4 x float> %m
+}
+
+define <2 x float> @fneg_lhs_lane_v2f32(<2 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: fneg_lhs_lane_v2f32:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.2s, #128, lsl #24
+; CHECK-NEXT: fmls v2.2s, v0.2s, v1.s[3]
+; CHECK-NEXT: fmov d0, d2
+; CHECK-NEXT: ret
+ %sp = shufflevector <4 x float> %b, <4 x float> poison, <2 x i32> <i32 3, i32 3>
+ %n = fneg <2 x float> %a
+ %m = fmul <2 x float> %n, %sp
+ ret <2 x float> %m
+}
+
+define <8 x half> @fneg_lhs_lane_v8f16(<8 x half> %a, <8 x half> %b) {
+; CHECK-SD-NOFP16-LABEL: fneg_lhs_lane_v8f16:
+; CHECK-SD-NOFP16: // %bb.0:
+; CHECK-SD-NOFP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-SD-NOFP16-NEXT: dup v1.8h, v1.h[3]
+; CHECK-SD-NOFP16-NEXT: fcvtl v3.4s, v1.4h
+; CHECK-SD-NOFP16-NEXT: fcvtl2 v1.4s, v1.8h
+; CHECK-SD-NOFP16-NEXT: eor v0.16b, v0.16b, v2.16b
+; CHECK-SD-NOFP16-NEXT: fcvtl v2.4s, v0.4h
+; CHECK-SD-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-SD-NOFP16-NEXT: fmul v2.4s, v2.4s, v3.4s
+; CHECK-SD-NOFP16-NEXT: fmul v1.4s, v0.4s, v1.4s
+; CHECK-SD-NOFP16-NEXT: fcvtn v0.4h, v2.4s
+; CHECK-SD-NOFP16-NEXT: fcvtn2 v0.8h, v1.4s
+; CHECK-SD-NOFP16-NEXT: ret
+;
+; CHECK-SD-FP16-LABEL: fneg_lhs_lane_v8f16:
+; CHECK-SD-FP16: // %bb.0:
+; CHECK-SD-FP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-SD-FP16-NEXT: fmls v2.8h, v0.8h, v1.h[3]
+; CHECK-SD-FP16-NEXT: mov v0.16b, v2.16b
+; CHECK-SD-FP16-NEXT: ret
+;
+; CHECK-GI-NOFP16-LABEL: fneg_lhs_lane_v8f16:
+; CHECK-GI-NOFP16: // %bb.0:
+; CHECK-GI-NOFP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-GI-NOFP16-NEXT: dup v1.4h, v1.h[3]
+; CHECK-GI-NOFP16-NEXT: eor v0.16b, v0.16b, v2.16b
+; CHECK-GI-NOFP16-NEXT: fcvtl v1.4s, v1.4h
+; CHECK-GI-NOFP16-NEXT: fcvtl v2.4s, v0.4h
+; CHECK-GI-NOFP16-NEXT: fcvtl2 v0.4s, v0.8h
+; CHECK-GI-NOFP16-NEXT: fmul v2.4s, v2.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT: fmul v1.4s, v0.4s, v1.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn v0.4h, v2.4s
+; CHECK-GI-NOFP16-NEXT: fcvtn2 v0.8h, v1.4s
+; CHECK-GI-NOFP16-NEXT: ret
+;
+; CHECK-GI-FP16-LABEL: fneg_lhs_lane_v8f16:
+; CHECK-GI-FP16: // %bb.0:
+; CHECK-GI-FP16-NEXT: movi v2.8h, #128, lsl #8
+; CHECK-GI-FP16-NEXT: fmls v2.8h, v0.8h, v1.h[3]
+; CHECK-GI-FP16-NEXT: mov v0.16b, v2.16b
+; CHECK-GI-FP16-NEXT: ret
+ %sp = shufflevector <8 x half> %b, <8 x half> poison, <8 x i32> <i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3, i32 3>
+ %n = fneg <8 x half> %a
+ %m = fmul <8 x half> %n, %sp
+ ret <8 x half> %m
+}
+
+define <4 x float> @lane_shared_with_fmla(<4 x float> %a, <4 x float> %b) {
+; CHECK-LABEL: lane_shared_with_fmla:
+; CHECK: // %bb.0:
+; CHECK-NEXT: movi v2.4s, #128, lsl #24
+; CHECK-NEXT: movi v3.4s, #128, lsl #24
+; CHECK-NEXT: dup v4.4s, v0.s[1]
+; CHECK-NEXT: dup v0.4s, v0.s[2]
+; CHECK-NEXT: fmls v3.4s, v4.4s, v1.s[2]
+; CHECK-NEXT: fmls v2.4s, v0.4s, v1.s[0]
+; CHECK-NEXT: fmla v3.4s, v0.4s, v1.s[1]
+; CHECK-NEXT: fmla v2.4s, v4.4s, v1.s[2]
+; CHECK-NEXT: fadd v0.4s, v3.4s, v2.4s
+; CHECK-NEXT: ret
+ %a1 = shufflevector <4 x float> %a, <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+ %a2 = shufflevector <4 x float> %a, <4 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+ %b0 = shufflevector <4 x float> %b, <4 x float> poison, <4 x i32> zeroinitializer
+ %b1 = shufflevector <4 x float> %b, <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+ %b2 = shufflevector <4 x float> %b, <4 x float> poison, <4 x i32> <i32 2, i32 2, i32 2, i32 2>
+ %n1 = fneg <4 x float> %a1
+ %t0 = fmul <4 x float> %b2, %n1
+ %x = call <4 x float> @llvm.fma.v4f32(<4 x float> %b1, <4 x float> %a2, <4 x float> %t0)
+ %n2 = fneg <4 x float> %a2
+ %t1 = fmul <4 x float> %b0, %n2
+ %y = call <4 x float> @llvm.fma.v4f32(<4 x float> %b2, <4 x float> %a1, <4 x float> %t1)
+ %r = fadd <4 x float> %x, %y
+ ret <4 x float> %r
+}
+
+; Negative test: the fneg feeds two multiplies, so the splat stays live.
+define void @neg_fmul_lane_multi_use(<4 x float> %a, <4 x float> %b, <4 x float> %c, ptr %p, ptr %q) {
+; CHECK-LABEL: neg_fmul_lane_multi_use:
+; CHECK: // %bb.0:
+; CHECK-NEXT: dup v1.4s, v1.s[1]
+; CHECK-NEXT: fneg v1.4s, v1.4s
+; CHECK-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: fmul v1.4s, v2.4s, v1.4s
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: str q1, [x1]
+; CHECK-NEXT: ret
+ %sp = shufflevector <4 x float> %b, <4 x float> poison, <4 x i32> <i32 1, i32 1, i32 1, i32 1>
+ %n = fneg <4 x float> %sp
+ %m1 = fmul <4 x float> %a, %n
+ %m2 = fmul <4 x float> %c, %n
+ store <4 x float> %m1, ptr %p
+ store <4 x float> %m2, ptr %q
+ ret void
+}
+
+; Negative test: -0.0 is not an encodable MOVI immediate for f64.
+define <2 x double> @fmul_neg_v2f64(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: fmul_neg_v2f64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fneg v1.2d, v1.2d
+; CHECK-NEXT: fmul v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: ret
+ %n = fneg <2 x double> %b
+ %m = fmul <2 x double> %a, %n
+ ret <2 x double> %m
+}
+
+; Negative test: -0.0 is not an encodable MOVI immediate for f64.
+define <2 x double> @neg_fmul_v2f64(<2 x double> %a, <2 x double> %b) {
+; CHECK-LABEL: neg_fmul_v2f64:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmul v0.2d, v0.2d, v1.2d
+; CHECK-NEXT: fneg v0.2d, v0.2d
+; CHECK-NEXT: ret
+ %m = fmul <2 x double> %a, %b
+ %n = fneg <2 x double> %m
+ ret <2 x double> %n
+}
+
+; Negative test: the fneg feeds two multiplies, so it stays live.
+define void @fmul_neg_multi_use(<4 x float> %a, <4 x float> %b, <4 x float> %c, ptr %p, ptr %q) {
+; CHECK-LABEL: fmul_neg_multi_use:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fneg v1.4s, v1.4s
+; CHECK-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: fmul v1.4s, v2.4s, v1.4s
+; CHECK-NEXT: str q0, [x0]
+; CHECK-NEXT: str q1, [x1]
+; CHECK-NEXT: ret
+ %n = fneg <4 x float> %b
+ %m1 = fmul <4 x float> %a, %n
+ %m2 = fmul <4 x float> %c, %n
+ store <4 x float> %m1, ptr %p
+ store <4 x float> %m2, ptr %q
+ ret void
+}
+
+; Negative test: the fneg has another use, so it stays live.
+define <4 x float> @fmul_neg_extra_use(<4 x float> %a, <4 x float> %b, ptr %p) {
+; CHECK-LABEL: fmul_neg_extra_use:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fneg v1.4s, v1.4s
+; CHECK-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: str q1, [x0]
+; CHECK-NEXT: ret
+ %n = fneg <4 x float> %b
+ store <4 x float> %n, ptr %p
+ %m = fmul <4 x float> %a, %n
+ ret <4 x float> %m
+}
+
+; Negative test: the fmul has another use, so it stays live.
+define <4 x float> @neg_fmul_extra_use(<4 x float> %a, <4 x float> %b, ptr %p) {
+; CHECK-LABEL: neg_fmul_extra_use:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmul v1.4s, v0.4s, v1.4s
+; CHECK-NEXT: fneg v0.4s, v1.4s
+; CHECK-NEXT: str q1, [x0]
+; CHECK-NEXT: ret
+ %m = fmul <4 x float> %a, %b
+ store <4 x float> %m, ptr %p
+ %n = fneg <4 x float> %m
+ ret <4 x float> %n
+}
+
+; Negative test: FMLS rounds -(x*y) once, which only matches FNEG of the
+; rounded product in the default rounding mode.
+define <4 x float> @fmul_neg_strict(<4 x float> %a, <4 x float> %b) #0 {
+; CHECK-LABEL: fmul_neg_strict:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fneg v1.4s, v1.4s
+; CHECK-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: ret
+ %n = fneg <4 x float> %b
+ %m = call <4 x float> @llvm.experimental.constrained.fmul.v4f32(<4 x float> %a, <4 x float> %n, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+ ret <4 x float> %m
+}
+
+; Negative test: FMLS rounds -(x*y) once, which only matches FNEG of the
+; rounded product in the default rounding mode.
+define <4 x float> @neg_fmul_strict(<4 x float> %a, <4 x float> %b) #0 {
+; CHECK-LABEL: neg_fmul_strict:
+; CHECK: // %bb.0:
+; CHECK-NEXT: fmul v0.4s, v0.4s, v1.4s
+; CHECK-NEXT: fneg v0.4s, v0.4s
+; CHECK-NEXT: ret
+ %m = call <4 x float> @llvm.experimental.constrained.fmul.v4f32(<4 x float> %a, <4 x float> %b, metadata !"round.dynamic", metadata !"fpexcept.strict") #0
+ %n = fneg <4 x float> %m
+ ret <4 x float> %n
+}
+
+attributes #0 = { strictfp }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-GI: {{.*}}
+; CHECK-SD: {{.*}}
More information about the llvm-commits
mailing list