[llvm] cfee494 - [AArch64][SVE] Extend predicated fma patterns to negative zero
David Green via llvm-commits
llvm-commits at lists.llvm.org
Wed Apr 12 07:53:29 PDT 2023
Author: David Green
Date: 2023-04-12T15:53:22+01:00
New Revision: cfee494fea097e97387bb3784fe257e4b0db5821
URL: https://github.com/llvm/llvm-project/commit/cfee494fea097e97387bb3784fe257e4b0db5821
DIFF: https://github.com/llvm/llvm-project/commit/cfee494fea097e97387bb3784fe257e4b0db5821.diff
LOG: [AArch64][SVE] Extend predicated fma patterns to negative zero
This extends the patterns added in D130564 for fma to also handle negative 0.0.
-0.0 is the identity element for fadd so comes up in vectorized loops.
The same basic idea applies to D130564, but nsz should no longer be needed for
the fadd case, and is for fsub (which is really only added for completeness).
Differential Revision: https://reviews.llvm.org/D147723
Added:
Modified:
llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
llvm/lib/Target/AArch64/SVEInstrFormats.td
llvm/test/CodeGen/AArch64/sve-fp-combine.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
index cc941600481b4..e83e58193ed9c 100644
--- a/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AArch64/AArch64ISelDAGToDAG.cpp
@@ -243,6 +243,18 @@ class AArch64DAGToDAGISel : public SelectionDAGISel {
return false;
}
+ bool SelectDupNegativeZero(SDValue N) {
+ switch(N->getOpcode()) {
+ case AArch64ISD::DUP:
+ case ISD::SPLAT_VECTOR: {
+ ConstantFPSDNode *Const = dyn_cast<ConstantFPSDNode>(N->getOperand(0));
+ return Const && Const->isZero() && Const->isNegative();
+ }
+ }
+
+ return false;
+ }
+
template<MVT::SimpleValueType VT>
bool SelectSVEAddSubImm(SDValue N, SDValue &Imm, SDValue &Shift) {
return SelectSVEAddSubImm(N, VT, Imm, Shift);
diff --git a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
index af3825eb3997f..f87bfbd0c9aac 100644
--- a/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
+++ b/llvm/lib/Target/AArch64/AArch64SVEInstrInfo.td
@@ -208,6 +208,10 @@ def AArch64fadd_p_nsz : PatFrag<(ops node:$op1, node:$op2, node:$op3),
(AArch64fadd_p node:$op1, node:$op2, node:$op3), [{
return N->getFlags().hasNoSignedZeros();
}]>;
+def AArch64fsub_p_nsz : PatFrag<(ops node:$op1, node:$op2, node:$op3),
+ (AArch64fsub_p node:$op1, node:$op2, node:$op3), [{
+ return N->getFlags().hasNoSignedZeros();
+}]>;
def SDT_AArch64Arith_Imm : SDTypeProfile<1, 3, [
SDTCisVec<0>, SDTCisVec<1>, SDTCisVec<2>, SDTCisVT<3,i32>,
@@ -416,10 +420,11 @@ def AArch64eor3 : PatFrags<(ops node:$op1, node:$op2, node:$op3),
[(int_aarch64_sve_eor3 node:$op1, node:$op2, node:$op3),
(xor node:$op1, (xor node:$op2, node:$op3))]>;
-class fma_patfrags<SDPatternOperator intrinsic, SDPatternOperator sdnode>
+class fma_patfrags<SDPatternOperator intrinsic, SDPatternOperator add_zero, SDPatternOperator add_negzero>
: PatFrags<(ops node:$pred, node:$op1, node:$op2, node:$op3),
[(intrinsic node:$pred, node:$op1, node:$op2, node:$op3),
- (sdnode (SVEAllActive), node:$op1, (vselect node:$pred, (AArch64fmul_p_oneuse (SVEAllActive), node:$op2, node:$op3), (SVEDup0)))],
+ (add_zero (SVEAllActive), node:$op1, (vselect node:$pred, (AArch64fmul_p_oneuse (SVEAllActive), node:$op2, node:$op3), (SVEDup0))),
+ (add_negzero (SVEAllActive), node:$op1, (vselect node:$pred, (AArch64fmul_p_oneuse (SVEAllActive), node:$op2, node:$op3), (SVEDupNeg0)))],
[{
if ((N->getOpcode() != AArch64ISD::FADD_PRED) &&
(N->getOpcode() != AArch64ISD::FSUB_PRED))
@@ -427,8 +432,8 @@ class fma_patfrags<SDPatternOperator intrinsic, SDPatternOperator sdnode>
return N->getFlags().hasAllowContract();
}]>;
-def AArch64fmla_m1 : fma_patfrags<int_aarch64_sve_fmla, AArch64fadd_p_nsz>;
-def AArch64fmls_m1 : fma_patfrags<int_aarch64_sve_fmls, AArch64fsub_p>;
+def AArch64fmla_m1 : fma_patfrags<int_aarch64_sve_fmla, AArch64fadd_p_nsz, AArch64fadd_p>;
+def AArch64fmls_m1 : fma_patfrags<int_aarch64_sve_fmls, AArch64fsub_p, AArch64fsub_p_nsz>;
def AArch64smax_m1 : EitherVSelectOrPassthruPatFrags<int_aarch64_sve_smax, AArch64smax_p>;
def AArch64umax_m1 : EitherVSelectOrPassthruPatFrags<int_aarch64_sve_umax, AArch64umax_p>;
diff --git a/llvm/lib/Target/AArch64/SVEInstrFormats.td b/llvm/lib/Target/AArch64/SVEInstrFormats.td
index 82561363c5339..d55c0351afcde 100644
--- a/llvm/lib/Target/AArch64/SVEInstrFormats.td
+++ b/llvm/lib/Target/AArch64/SVEInstrFormats.td
@@ -435,6 +435,7 @@ multiclass SVE_1_Op_PassthruUndef_Round_Pat<ValueType vtd, SDPatternOperator op,
}
def SVEDup0 : ComplexPattern<vAny, 0, "SelectDupZero", []>;
+def SVEDupNeg0 : ComplexPattern<vAny, 0, "SelectDupNegativeZero", []>;
class SVE_1_Op_PassthruZero_Pat<ValueType vtd, SDPatternOperator op, ValueType vt1,
ValueType vt2, Instruction inst>
diff --git a/llvm/test/CodeGen/AArch64/sve-fp-combine.ll b/llvm/test/CodeGen/AArch64/sve-fp-combine.ll
index db364814e3600..58242b5676aa9 100644
--- a/llvm/test/CodeGen/AArch64/sve-fp-combine.ll
+++ b/llvm/test/CodeGen/AArch64/sve-fp-combine.ll
@@ -1116,11 +1116,7 @@ define <vscale x 2 x double> @fsub_sel_fmul_d_nsz(<vscale x 2 x double> %a, <vsc
define <vscale x 8 x half> @fadd_sel_fmul_h_negzero(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x i1> %mask) {
; CHECK-LABEL: fadd_sel_fmul_h_negzero:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #32768 // =0x8000
-; CHECK-NEXT: fmul z1.h, z1.h, z2.h
-; CHECK-NEXT: mov z2.h, w8
-; CHECK-NEXT: sel z1.h, p0, z1.h, z2.h
-; CHECK-NEXT: fadd z0.h, z0.h, z1.h
+; CHECK-NEXT: fmla z0.h, p0/m, z1.h, z2.h
; CHECK-NEXT: ret
%fmul = fmul <vscale x 8 x half> %b, %c
%nz = fneg <vscale x 8 x half> zeroinitializer
@@ -1132,11 +1128,7 @@ define <vscale x 8 x half> @fadd_sel_fmul_h_negzero(<vscale x 8 x half> %a, <vsc
define <vscale x 4 x float> @fadd_sel_fmul_s_negzero(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x i1> %mask) {
; CHECK-LABEL: fadd_sel_fmul_s_negzero:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #-2147483648 // =0x80000000
-; CHECK-NEXT: fmul z1.s, z1.s, z2.s
-; CHECK-NEXT: mov z2.s, w8
-; CHECK-NEXT: sel z1.s, p0, z1.s, z2.s
-; CHECK-NEXT: fadd z0.s, z0.s, z1.s
+; CHECK-NEXT: fmla z0.s, p0/m, z1.s, z2.s
; CHECK-NEXT: ret
%fmul = fmul <vscale x 4 x float> %b, %c
%nz = fneg <vscale x 4 x float> zeroinitializer
@@ -1148,11 +1140,7 @@ define <vscale x 4 x float> @fadd_sel_fmul_s_negzero(<vscale x 4 x float> %a, <v
define <vscale x 2 x double> @fadd_sel_fmul_d_negzero(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x i1> %mask) {
; CHECK-LABEL: fadd_sel_fmul_d_negzero:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: fmul z1.d, z1.d, z2.d
-; CHECK-NEXT: mov z2.d, x8
-; CHECK-NEXT: sel z1.d, p0, z1.d, z2.d
-; CHECK-NEXT: fadd z0.d, z0.d, z1.d
+; CHECK-NEXT: fmla z0.d, p0/m, z1.d, z2.d
; CHECK-NEXT: ret
%fmul = fmul <vscale x 2 x double> %b, %c
%nz = fneg <vscale x 2 x double> zeroinitializer
@@ -1214,11 +1202,7 @@ define <vscale x 2 x double> @fsub_sel_fmul_d_negzero(<vscale x 2 x double> %a,
define <vscale x 8 x half> @fadd_sel_fmul_h_negzero_nsz(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x i1> %mask) {
; CHECK-LABEL: fadd_sel_fmul_h_negzero_nsz:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #32768 // =0x8000
-; CHECK-NEXT: fmul z1.h, z1.h, z2.h
-; CHECK-NEXT: mov z2.h, w8
-; CHECK-NEXT: sel z1.h, p0, z1.h, z2.h
-; CHECK-NEXT: fadd z0.h, z0.h, z1.h
+; CHECK-NEXT: fmla z0.h, p0/m, z1.h, z2.h
; CHECK-NEXT: ret
%fmul = fmul <vscale x 8 x half> %b, %c
%nz = fneg <vscale x 8 x half> zeroinitializer
@@ -1230,11 +1214,7 @@ define <vscale x 8 x half> @fadd_sel_fmul_h_negzero_nsz(<vscale x 8 x half> %a,
define <vscale x 4 x float> @fadd_sel_fmul_s_negzero_nsz(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x i1> %mask) {
; CHECK-LABEL: fadd_sel_fmul_s_negzero_nsz:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #-2147483648 // =0x80000000
-; CHECK-NEXT: fmul z1.s, z1.s, z2.s
-; CHECK-NEXT: mov z2.s, w8
-; CHECK-NEXT: sel z1.s, p0, z1.s, z2.s
-; CHECK-NEXT: fadd z0.s, z0.s, z1.s
+; CHECK-NEXT: fmla z0.s, p0/m, z1.s, z2.s
; CHECK-NEXT: ret
%fmul = fmul <vscale x 4 x float> %b, %c
%nz = fneg <vscale x 4 x float> zeroinitializer
@@ -1246,11 +1226,7 @@ define <vscale x 4 x float> @fadd_sel_fmul_s_negzero_nsz(<vscale x 4 x float> %a
define <vscale x 2 x double> @fadd_sel_fmul_d_negzero_nsz(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x i1> %mask) {
; CHECK-LABEL: fadd_sel_fmul_d_negzero_nsz:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: fmul z1.d, z1.d, z2.d
-; CHECK-NEXT: mov z2.d, x8
-; CHECK-NEXT: sel z1.d, p0, z1.d, z2.d
-; CHECK-NEXT: fadd z0.d, z0.d, z1.d
+; CHECK-NEXT: fmla z0.d, p0/m, z1.d, z2.d
; CHECK-NEXT: ret
%fmul = fmul <vscale x 2 x double> %b, %c
%nz = fneg <vscale x 2 x double> zeroinitializer
@@ -1262,11 +1238,7 @@ define <vscale x 2 x double> @fadd_sel_fmul_d_negzero_nsz(<vscale x 2 x double>
define <vscale x 8 x half> @fsub_sel_fmul_h_negzero_nsz(<vscale x 8 x half> %a, <vscale x 8 x half> %b, <vscale x 8 x half> %c, <vscale x 8 x i1> %mask) {
; CHECK-LABEL: fsub_sel_fmul_h_negzero_nsz:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #32768 // =0x8000
-; CHECK-NEXT: fmul z1.h, z1.h, z2.h
-; CHECK-NEXT: mov z2.h, w8
-; CHECK-NEXT: sel z1.h, p0, z1.h, z2.h
-; CHECK-NEXT: fsub z0.h, z0.h, z1.h
+; CHECK-NEXT: fmls z0.h, p0/m, z1.h, z2.h
; CHECK-NEXT: ret
%fmul = fmul <vscale x 8 x half> %b, %c
%nz = fneg <vscale x 8 x half> zeroinitializer
@@ -1278,11 +1250,7 @@ define <vscale x 8 x half> @fsub_sel_fmul_h_negzero_nsz(<vscale x 8 x half> %a,
define <vscale x 4 x float> @fsub_sel_fmul_s_negzero_nsz(<vscale x 4 x float> %a, <vscale x 4 x float> %b, <vscale x 4 x float> %c, <vscale x 4 x i1> %mask) {
; CHECK-LABEL: fsub_sel_fmul_s_negzero_nsz:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov w8, #-2147483648 // =0x80000000
-; CHECK-NEXT: fmul z1.s, z1.s, z2.s
-; CHECK-NEXT: mov z2.s, w8
-; CHECK-NEXT: sel z1.s, p0, z1.s, z2.s
-; CHECK-NEXT: fsub z0.s, z0.s, z1.s
+; CHECK-NEXT: fmls z0.s, p0/m, z1.s, z2.s
; CHECK-NEXT: ret
%fmul = fmul <vscale x 4 x float> %b, %c
%nz = fneg <vscale x 4 x float> zeroinitializer
@@ -1294,11 +1262,7 @@ define <vscale x 4 x float> @fsub_sel_fmul_s_negzero_nsz(<vscale x 4 x float> %a
define <vscale x 2 x double> @fsub_sel_fmul_d_negzero_nsz(<vscale x 2 x double> %a, <vscale x 2 x double> %b, <vscale x 2 x double> %c, <vscale x 2 x i1> %mask) {
; CHECK-LABEL: fsub_sel_fmul_d_negzero_nsz:
; CHECK: // %bb.0:
-; CHECK-NEXT: mov x8, #-9223372036854775808 // =0x8000000000000000
-; CHECK-NEXT: fmul z1.d, z1.d, z2.d
-; CHECK-NEXT: mov z2.d, x8
-; CHECK-NEXT: sel z1.d, p0, z1.d, z2.d
-; CHECK-NEXT: fsub z0.d, z0.d, z1.d
+; CHECK-NEXT: fmls z0.d, p0/m, z1.d, z2.d
; CHECK-NEXT: ret
%fmul = fmul <vscale x 2 x double> %b, %c
%nz = fneg <vscale x 2 x double> zeroinitializer
More information about the llvm-commits
mailing list