[llvm] 8f8016f - [NVPTX] Add patterns for fma.relu.{f16|f16x2|bf16|bf16x2} (#114977)
via llvm-commits
llvm-commits at lists.llvm.org
Mon Nov 18 07:29:21 PST 2024
Author: Hugh Delaney
Date: 2024-11-18T15:29:17Z
New Revision: 8f8016fe66dd260b03a4d1c2b50636e36e02942b
URL: https://github.com/llvm/llvm-project/commit/8f8016fe66dd260b03a4d1c2b50636e36e02942b
DIFF: https://github.com/llvm/llvm-project/commit/8f8016fe66dd260b03a4d1c2b50636e36e02942b.diff
LOG: [NVPTX] Add patterns for fma.relu.{f16|f16x2|bf16|bf16x2} (#114977)
Add patterns to lower `fmaxnum(fma(a, b, c), 0)` to `fma.rn{.ftz}.relu`
for `f16`, `f16x2`, `bf16`, `bf16x2` types, when `nnan` is used.
`fma_relu` honours `NaN`, so the substitution is only made if the `fma`
is `nnan`, since `fmaxnum` returns the non NaN argument when passed a
NaN value.
This patch also removes some `bf16` ftz instructions since `FTZ` is not
supported with the `bf16` type, according to the PTX ISA docs.
Added:
llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
Modified:
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
Removed:
################################################################################
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index a16935dcbb93be..64f437fd4e4d2d 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -418,12 +418,6 @@ multiclass F3_fma_component<string OpcStr, SDNode OpNode> {
!strconcat(OpcStr, ".f16x2 \t$dst, $a, $b;"),
[(set Int32Regs:$dst, (OpNode (v2f16 Int32Regs:$a), (v2f16 Int32Regs:$b)))]>,
Requires<[useFP16Math, allowFMA]>;
- def bf16rr_ftz :
- NVPTXInst<(outs Int16Regs:$dst),
- (ins Int16Regs:$a, Int16Regs:$b),
- !strconcat(OpcStr, ".ftz.bf16 \t$dst, $a, $b;"),
- [(set Int16Regs:$dst, (OpNode (bf16 Int16Regs:$a), (bf16 Int16Regs:$b)))]>,
- Requires<[hasBF16Math, allowFMA, doF32FTZ]>;
def bf16rr :
NVPTXInst<(outs Int16Regs:$dst),
(ins Int16Regs:$a, Int16Regs:$b),
@@ -431,12 +425,6 @@ multiclass F3_fma_component<string OpcStr, SDNode OpNode> {
[(set Int16Regs:$dst, (OpNode (bf16 Int16Regs:$a), (bf16 Int16Regs:$b)))]>,
Requires<[hasBF16Math, allowFMA]>;
- def bf16x2rr_ftz :
- NVPTXInst<(outs Int32Regs:$dst),
- (ins Int32Regs:$a, Int32Regs:$b),
- !strconcat(OpcStr, ".ftz.bf16x2 \t$dst, $a, $b;"),
- [(set (v2bf16 Int32Regs:$dst), (OpNode (v2bf16 Int32Regs:$a), (v2bf16 Int32Regs:$b)))]>,
- Requires<[hasBF16Math, allowFMA, doF32FTZ]>;
def bf16x2rr :
NVPTXInst<(outs Int32Regs:$dst),
(ins Int32Regs:$a, Int32Regs:$b),
@@ -1423,9 +1411,7 @@ defm FMA16_ftz : FMA_F16<"fma.rn.ftz.f16", f16, Int16Regs, doF32FTZ>;
defm FMA16 : FMA_F16<"fma.rn.f16", f16, Int16Regs, True>;
defm FMA16x2_ftz : FMA_F16<"fma.rn.ftz.f16x2", v2f16, Int32Regs, doF32FTZ>;
defm FMA16x2 : FMA_F16<"fma.rn.f16x2", v2f16, Int32Regs, True>;
-defm BFMA16_ftz : FMA_BF16<"fma.rn.ftz.bf16", bf16, Int16Regs, doF32FTZ>;
defm BFMA16 : FMA_BF16<"fma.rn.bf16", bf16, Int16Regs, True>;
-defm BFMA16x2_ftz : FMA_BF16<"fma.rn.ftz.bf16x2", v2bf16, Int32Regs, doF32FTZ>;
defm BFMA16x2 : FMA_BF16<"fma.rn.bf16x2", v2bf16, Int32Regs, True>;
defm FMA32_ftz : FMA<"fma.rn.ftz.f32", Float32Regs, f32imm, doF32FTZ>;
defm FMA32 : FMA<"fma.rn.f32", Float32Regs, f32imm, True>;
@@ -3959,3 +3945,54 @@ def atomic_thread_fence_seq_cst_cta :
def atomic_thread_fence_acq_rel_cta :
NVPTXInst<(outs), (ins), "fence.acq_rel.cta;", []>,
Requires<[hasPTX<60>, hasSM<70>]>;
+
+def fpimm_any_zero : FPImmLeaf<fAny, [{
+ return Imm.isZero();
+}]>;
+
+def fpimm_positive_zero_v2f16 : PatFrag<(ops), (v2f16 (bitconvert (i32 0)))>;
+def fpimm_positive_zero_v2bf16 : PatFrag<(ops), (v2bf16 (bitconvert (i32 0)))>;
+
+// Perform substitution if fma only has one use, and also if instruction has
+// nnan instruction flag or if the TM has NoNaNsFPMath
+def NVPTX_fma_oneuse_and_nnan : PatFrag<(ops node:$a, node:$b, node:$c),
+ (fma node:$a, node:$b, node:$c), [{
+ return N->hasOneUse() &&
+ (N->getFlags().hasNoNaNs() || TM.Options.NoNaNsFPMath);
+}]>;
+// fmaxnum will
diff erentiate between signed and unsigned zeros soon, so this
+// PatFrag is for a fmaxnum node with nsz
+def NVPTX_fmaxnum_nsz : PatFrag<(ops node:$a, node:$b),
+ (fmaxnum node:$a, node:$b), [{
+ return N->getFlags().hasNoSignedZeros() || TM.Options.NoSignedZerosFPMath;
+}]>;
+
+class NVPTXInst_rrr<RegisterClass RC, string Instruction, list<Predicate> Preds>
+ : NVPTXInst<(outs RC:$dst), (ins RC:$a, RC:$b, RC:$c),
+ !strconcat(Instruction, "\t$dst, $a, $b, $c;"), []>,
+ Requires<Preds>;
+
+def FMARELU_F16 : NVPTXInst_rrr<Int16Regs, "fma.rn.relu.f16", [useFP16Math, hasPTX<70>, hasSM<80>]>;
+def FMARELU_F16_FTZ : NVPTXInst_rrr<Int16Regs, "fma.rn.ftz.relu.f16", [useFP16Math, hasPTX<70>, hasSM<80>]>;
+def FMARELU_BF16 : NVPTXInst_rrr<Int16Regs, "fma.rn.relu.bf16", [hasBF16Math, hasPTX<70>, hasSM<80>]>;
+def FMARELU_F16X2 : NVPTXInst_rrr<Int32Regs, "fma.rn.relu.f16x2", [useFP16Math, hasPTX<70>, hasSM<80>]>;
+def FMARELU_F16X2_FTZ : NVPTXInst_rrr<Int32Regs, "fma.rn.ftz.relu.f16x2", [useFP16Math, hasPTX<70>, hasSM<80>]>;
+def FMARELU_BF16X2 : NVPTXInst_rrr<Int32Regs, "fma.rn.relu.bf16x2", [hasBF16Math, hasPTX<70>, hasSM<80>]>;
+
+// FTZ
+def : Pat<(f16 (NVPTX_fmaxnum_nsz (NVPTX_fma_oneuse_and_nnan Int16Regs:$a, Int16Regs:$b, Int16Regs:$c), fpimm_any_zero)),
+ (FMARELU_F16_FTZ Int16Regs:$a, Int16Regs:$b, Int16Regs:$c)>,
+ Requires<[doF32FTZ]>;
+def : Pat<(v2f16 (NVPTX_fmaxnum_nsz (NVPTX_fma_oneuse_and_nnan Int32Regs:$a, Int32Regs:$b, Int32Regs:$c), fpimm_positive_zero_v2f16)),
+ (FMARELU_F16X2_FTZ Int32Regs:$a, Int32Regs:$b, Int32Regs:$c)>,
+ Requires<[doF32FTZ]>;
+
+// NO FTZ
+def : Pat<(f16 (NVPTX_fmaxnum_nsz (NVPTX_fma_oneuse_and_nnan Int16Regs:$a, Int16Regs:$b, Int16Regs:$c), fpimm_any_zero)),
+ (FMARELU_F16 Int16Regs:$a, Int16Regs:$b, Int16Regs:$c)>;
+def : Pat<(bf16 (NVPTX_fmaxnum_nsz (NVPTX_fma_oneuse_and_nnan Int16Regs:$a, Int16Regs:$b, Int16Regs:$c), fpimm_any_zero)),
+ (FMARELU_BF16 Int16Regs:$a, Int16Regs:$b, Int16Regs:$c)>;
+def : Pat<(v2f16 (NVPTX_fmaxnum_nsz (NVPTX_fma_oneuse_and_nnan Int32Regs:$a, Int32Regs:$b, Int32Regs:$c), fpimm_positive_zero_v2f16)),
+ (FMARELU_F16X2 Int32Regs:$a, Int32Regs:$b, Int32Regs:$c)>;
+def : Pat<(v2bf16 (NVPTX_fmaxnum_nsz (NVPTX_fma_oneuse_and_nnan Int32Regs:$a, Int32Regs:$b, Int32Regs:$c), fpimm_positive_zero_v2bf16)),
+ (FMARELU_BF16X2 Int32Regs:$a, Int32Regs:$b, Int32Regs:$c)>;
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll b/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
new file mode 100644
index 00000000000000..8cc4548f6e85e0
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-contract.ll
@@ -0,0 +1,1269 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_80 | %ptxas-verify -arch=sm_80 %}
+
+; Using FTZ should emit fma.ftz.relu for f16, not for bf16
+; RUN: llc < %s -denormal-fp-math-f32=preserve-sign -march=nvptx64 -mcpu=sm_80 | FileCheck %s --check-prefixes=CHECK-FTZ
+; RUN: %if ptxas %{ llc < %s -denormal-fp-math-f32=preserve-sign -march=nvptx64 -mcpu=sm_80 | %ptxas-verify -arch=sm_80 %}
+
+; SM < 80 or (which needs PTX version >= 70) should not emit fma{.ftz}.relu
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 | FileCheck %s --check-prefixes=CHECK-SM70
+
+define half @fma_f16_expanded_no_nans(half %a, half %b, half %c) #0 {
+; CHECK-LABEL: fma_f16_expanded_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_expanded_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_expanded_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<2>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs5;
+; CHECK-SM70-NEXT: selp.b16 %rs6, %rs4, 0x0000, %p1;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs6;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul half %a, %b
+ %2 = fadd half %1, %c
+ %3 = fcmp ogt half %2, 0.0
+ %4 = select i1 %3, half %2, half 0.0
+ ret half %4
+}
+
+; FMA relu shouldn't be selected if the FMA operation has multiple uses
+define half @fma_f16_expanded_no_nans_multiple_uses_of_fma(half %a, half %b, half %c) #0 {
+; CHECK-LABEL: fma_f16_expanded_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-NEXT: max.f16 %rs6, %rs4, %rs5;
+; CHECK-NEXT: mov.b16 %rs7, 0x4700;
+; CHECK-NEXT: add.f16 %rs8, %rs4, %rs7;
+; CHECK-NEXT: add.f16 %rs9, %rs6, %rs8;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs9;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<10>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-FTZ-NEXT: max.ftz.f16 %rs6, %rs4, %rs5;
+; CHECK-FTZ-NEXT: mov.b16 %rs7, 0x4700;
+; CHECK-FTZ-NEXT: add.ftz.f16 %rs8, %rs4, %rs7;
+; CHECK-FTZ-NEXT: add.ftz.f16 %rs9, %rs6, %rs8;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs9;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<2>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<10>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs5;
+; CHECK-SM70-NEXT: selp.b16 %rs6, %rs4, 0x0000, %p1;
+; CHECK-SM70-NEXT: mov.b16 %rs7, 0x4700;
+; CHECK-SM70-NEXT: add.f16 %rs8, %rs4, %rs7;
+; CHECK-SM70-NEXT: add.f16 %rs9, %rs6, %rs8;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs9;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul half %a, %b
+ %2 = fadd half %1, %c
+ %3 = fcmp ogt half %2, 0.0
+ %4 = select i1 %3, half %2, half 0.0
+ %5 = fadd half %2, 7.0
+ %6 = fadd half %4, %5
+ ret half %6
+}
+
+define half @fma_f16_expanded_unsafe_with_nans(half %a, half %b, half %c) #1 {
+; CHECK-LABEL: fma_f16_expanded_unsafe_with_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_unsafe_with_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_unsafe_with_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_unsafe_with_nans_param_2];
+; CHECK-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-NEXT: max.f16 %rs6, %rs4, %rs5;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs6;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_expanded_unsafe_with_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<7>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_unsafe_with_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_unsafe_with_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_unsafe_with_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-FTZ-NEXT: max.ftz.f16 %rs6, %rs4, %rs5;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs6;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_expanded_unsafe_with_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<2>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_unsafe_with_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_unsafe_with_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_unsafe_with_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs5;
+; CHECK-SM70-NEXT: selp.b16 %rs6, %rs4, 0x0000, %p1;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs6;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul half %a, %b
+ %2 = fadd half %1, %c
+ %3 = fcmp ogt half %2, 0.0
+ %4 = select i1 %3, half %2, half 0.0
+ ret half %4
+}
+
+define half @fma_f16_expanded_maxnum_no_nans(half %a, half %b, half %c) #0 {
+; CHECK-LABEL: fma_f16_expanded_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_maxnum_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_expanded_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_expanded_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .b16 %rs<6>;
+; CHECK-SM70-NEXT: .reg .f32 %f<3>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: cvt.f32.f16 %f1, %rs4;
+; CHECK-SM70-NEXT: max.f32 %f2, %f1, 0f00000000;
+; CHECK-SM70-NEXT: cvt.rn.f16.f32 %rs5, %f2;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs5;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul half %a, %b
+ %2 = fadd half %1, %c
+ %3 = call half @llvm.maxnum.f16(half %2, half 0.0)
+ ret half %3
+}
+
+define bfloat @fma_bf16_expanded_unsafe_with_nans(bfloat %a, bfloat %b, bfloat %c) #1 {
+; CHECK-LABEL: fma_bf16_expanded_unsafe_with_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_unsafe_with_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_unsafe_with_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_unsafe_with_nans_param_2];
+; CHECK-NEXT: fma.rn.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-NEXT: max.bf16 %rs6, %rs4, %rs5;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs6;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_expanded_unsafe_with_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<7>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_unsafe_with_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_unsafe_with_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_unsafe_with_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-FTZ-NEXT: max.bf16 %rs6, %rs4, %rs5;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs6;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_expanded_unsafe_with_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<4>;
+; CHECK-SM70-NEXT: .reg .b32 %r<14>;
+; CHECK-SM70-NEXT: .reg .f32 %f<6>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_expanded_unsafe_with_nans_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_expanded_unsafe_with_nans_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_expanded_unsafe_with_nans_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %f5, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs1, 0x0000, %p2;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul bfloat %a, %b
+ %2 = fadd bfloat %1, %c
+ %3 = fcmp ogt bfloat %2, 0.0
+ %4 = select i1 %3, bfloat %2, bfloat 0.0
+ ret bfloat %4
+}
+
+define bfloat @fma_bf16_expanded_no_nans(bfloat %a, bfloat %b, bfloat %c) #0 {
+; CHECK-LABEL: fma_bf16_expanded_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_expanded_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_expanded_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<4>;
+; CHECK-SM70-NEXT: .reg .b32 %r<14>;
+; CHECK-SM70-NEXT: .reg .f32 %f<6>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_expanded_no_nans_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_expanded_no_nans_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %f5, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs1, 0x0000, %p2;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul bfloat %a, %b
+ %2 = fadd bfloat %1, %c
+ %3 = fcmp ogt bfloat %2, 0.0
+ %4 = select i1 %3, bfloat %2, bfloat 0.0
+ ret bfloat %4
+}
+
+; FMA relu shouldn't be selected if the FMA operation has multiple uses
+define bfloat @fma_bf16_expanded_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloat %c) #0 {
+; CHECK-LABEL: fma_bf16_expanded_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<12>;
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .f32 %f<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-NEXT: max.bf16 %rs6, %rs4, %rs5;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs4;
+; CHECK-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-NEXT: mov.b32 %f1, %r2;
+; CHECK-NEXT: add.f32 %f2, %f1, 0f40E00000;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs8, %f2;
+; CHECK-NEXT: cvt.u32.u16 %r3, %rs6;
+; CHECK-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-NEXT: mov.b32 %f3, %r4;
+; CHECK-NEXT: cvt.u32.u16 %r5, %rs8;
+; CHECK-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-NEXT: mov.b32 %f4, %r6;
+; CHECK-NEXT: add.f32 %f5, %f3, %f4;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs11, %f5;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs11;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<12>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<7>;
+; CHECK-FTZ-NEXT: .reg .f32 %f<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-FTZ-NEXT: max.bf16 %rs6, %rs4, %rs5;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r1, %rs4;
+; CHECK-FTZ-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f1, %r2;
+; CHECK-FTZ-NEXT: add.ftz.f32 %f2, %f1, 0f40E00000;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs8, %f2;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r3, %rs6;
+; CHECK-FTZ-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f3, %r4;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r5, %rs8;
+; CHECK-FTZ-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f4, %r6;
+; CHECK-FTZ-NEXT: add.ftz.f32 %f5, %f3, %f4;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs11, %f5;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs11;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<5>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
+; CHECK-SM70-NEXT: .reg .b32 %r<29>;
+; CHECK-SM70-NEXT: .reg .f32 %f<10>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %f5, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs1, 0x0000, %p2;
+; CHECK-SM70-NEXT: add.f32 %f6, %f5, 0f40E00000;
+; CHECK-SM70-NEXT: mov.b32 %r14, %f6;
+; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
+; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %f6, %f6;
+; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: and.b32 %r22, %r19, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f8, %r22;
+; CHECK-SM70-NEXT: add.f32 %f9, %f7, %f8;
+; CHECK-SM70-NEXT: mov.b32 %r23, %f9;
+; CHECK-SM70-NEXT: bfe.u32 %r24, %r23, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, %r23;
+; CHECK-SM70-NEXT: add.s32 %r26, %r25, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %f9, %f9;
+; CHECK-SM70-NEXT: or.b32 %r27, %r23, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r28, %r27, %r26, %p4;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs5}, %r28; }
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs5;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul bfloat %a, %b
+ %2 = fadd bfloat %1, %c
+ %3 = fcmp ogt bfloat %2, 0.0
+ %4 = select i1 %3, bfloat %2, bfloat 0.0
+ %5 = fadd bfloat %2, 7.0
+ %6 = fadd bfloat %4, %5
+ ret bfloat %6
+}
+
+define bfloat @fma_bf16_expanded_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) #0 {
+; CHECK-LABEL: fma_bf16_expanded_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_maxnum_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_expanded_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_expanded_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
+; CHECK-SM70-NEXT: .reg .b32 %r<20>;
+; CHECK-SM70-NEXT: .reg .f32 %f<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_expanded_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_expanded_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: max.f32 %f6, %f5, 0f00000000;
+; CHECK-SM70-NEXT: mov.b32 %r14, %f6;
+; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
+; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f6, %f6;
+; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r19; }
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul bfloat %a, %b
+ %2 = fadd bfloat %1, %c
+ %3 = call bfloat @llvm.maxnum.bf16(bfloat %2, bfloat 0.0)
+ ret bfloat %3
+}
+
+define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
+; CHECK-LABEL: fma_f16x2_expanded_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_expanded_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_expanded_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
+; CHECK-SM70-NEXT: .reg .b32 %r<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 %r5, 0;
+; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs2, 0x0000, %p2;
+; CHECK-SM70-NEXT: selp.b16 %rs4, %rs1, 0x0000, %p1;
+; CHECK-SM70-NEXT: mov.b32 %r6, {%rs4, %rs3};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul <2 x half> %a, %b
+ %2 = fadd <2 x half> %1, %c
+ %3 = fcmp ogt <2 x half> %2, <half 0.0, half 0.0>
+ %4 = select <2 x i1> %3, <2 x half> %2, <2 x half> <half 0.0, half 0.0>
+ ret <2 x half> %4
+}
+
+; FMA relu shouldn't be selected if the FMA operation has multiple uses
+define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
+; CHECK-LABEL: fma_f16x2_expanded_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: mov.b32 %r5, 0;
+; CHECK-NEXT: max.f16x2 %r6, %r4, %r5;
+; CHECK-NEXT: mov.b32 %r7, 1191200512;
+; CHECK-NEXT: add.f16x2 %r8, %r4, %r7;
+; CHECK-NEXT: add.f16x2 %r9, %r6, %r8;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<10>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
+; CHECK-FTZ-NEXT: max.ftz.f16x2 %r6, %r4, %r5;
+; CHECK-FTZ-NEXT: mov.b32 %r7, 1191200512;
+; CHECK-FTZ-NEXT: add.ftz.f16x2 %r8, %r4, %r7;
+; CHECK-FTZ-NEXT: add.ftz.f16x2 %r9, %r6, %r8;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
+; CHECK-SM70-NEXT: .reg .b32 %r<10>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 %r5, 0;
+; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs2, 0x0000, %p2;
+; CHECK-SM70-NEXT: selp.b16 %rs4, %rs1, 0x0000, %p1;
+; CHECK-SM70-NEXT: mov.b32 %r6, {%rs4, %rs3};
+; CHECK-SM70-NEXT: mov.b32 %r7, 1191200512;
+; CHECK-SM70-NEXT: add.f16x2 %r8, %r4, %r7;
+; CHECK-SM70-NEXT: add.f16x2 %r9, %r6, %r8;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul <2 x half> %a, %b
+ %2 = fadd <2 x half> %1, %c
+ %3 = fcmp ogt <2 x half> %2, <half 0.0, half 0.0>
+ %4 = select <2 x i1> %3, <2 x half> %2, <2 x half> <half 0.0, half 0.0>
+ %5 = fadd <2 x half> %2, <half 7.0, half 7.0>
+ %6 = fadd <2 x half> %4, %5
+ ret <2 x half> %6
+}
+
+define <2 x half> @fma_f16x2_expanded_unsafe_with_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) #1 {
+; CHECK-LABEL: fma_f16x2_expanded_unsafe_with_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_unsafe_with_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_0];
+; CHECK-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: mov.b32 %r5, 0;
+; CHECK-NEXT: max.f16x2 %r6, %r4, %r5;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_expanded_unsafe_with_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<7>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_unsafe_with_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
+; CHECK-FTZ-NEXT: max.ftz.f16x2 %r6, %r4, %r5;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_expanded_unsafe_with_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
+; CHECK-SM70-NEXT: .reg .b32 %r<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_unsafe_with_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_unsafe_with_nans_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 %r5, 0;
+; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs2, 0x0000, %p2;
+; CHECK-SM70-NEXT: selp.b16 %rs4, %rs1, 0x0000, %p1;
+; CHECK-SM70-NEXT: mov.b32 %r6, {%rs4, %rs3};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul <2 x half> %a, %b
+ %2 = fadd <2 x half> %1, %c
+ %3 = fcmp ogt <2 x half> %2, <half 0.0, half 0.0>
+ %4 = select <2 x i1> %3, <2 x half> %2, <2 x half> <half 0.0, half 0.0>
+ ret <2 x half> %4
+}
+
+define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
+; CHECK-LABEL: fma_f16x2_expanded_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_expanded_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_expanded_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
+; CHECK-SM70-NEXT: .reg .b32 %r<6>;
+; CHECK-SM70-NEXT: .reg .f32 %f<5>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM70-NEXT: cvt.f32.f16 %f1, %rs2;
+; CHECK-SM70-NEXT: max.f32 %f2, %f1, 0f00000000;
+; CHECK-SM70-NEXT: cvt.rn.f16.f32 %rs3, %f2;
+; CHECK-SM70-NEXT: cvt.f32.f16 %f3, %rs1;
+; CHECK-SM70-NEXT: max.f32 %f4, %f3, 0f00000000;
+; CHECK-SM70-NEXT: cvt.rn.f16.f32 %rs4, %f4;
+; CHECK-SM70-NEXT: mov.b32 %r5, {%rs4, %rs3};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r5;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul <2 x half> %a, %b
+ %2 = fadd <2 x half> %1, %c
+ %3 = call <2 x half> @llvm.maxnum.f16x2(<2 x half> %2, <2 x half> <half 0.0, half 0.0>)
+ ret <2 x half> %3
+}
+
+define <2 x bfloat> @fma_bf16x2_expanded_unsafe_with_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #1 {
+; CHECK-LABEL: fma_bf16x2_expanded_unsafe_with_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_unsafe_with_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
+; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: mov.b32 %r5, 0;
+; CHECK-NEXT: max.bf16x2 %r6, %r4, %r5;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_expanded_unsafe_with_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<7>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_unsafe_with_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
+; CHECK-FTZ-NEXT: max.bf16x2 %r6, %r4, %r5;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_expanded_unsafe_with_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<5>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<19>;
+; CHECK-SM70-NEXT: .reg .b32 %r<31>;
+; CHECK-SM70-NEXT: .reg .f32 %f<11>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_unsafe_with_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_unsafe_with_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_unsafe_with_nans_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs10}, %r15; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r27; }
+; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %f9, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f10, %r29;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %f10, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs17, %rs15, 0x0000, %p4;
+; CHECK-SM70-NEXT: selp.b16 %rs18, %rs10, 0x0000, %p3;
+; CHECK-SM70-NEXT: mov.b32 %r30, {%rs18, %rs17};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul <2 x bfloat> %a, %b
+ %2 = fadd <2 x bfloat> %1, %c
+ %3 = fcmp ogt <2 x bfloat> %2, <bfloat 0.0, bfloat 0.0>
+ %4 = select <2 x i1> %3, <2 x bfloat> %2, <2 x bfloat> <bfloat 0.0, bfloat 0.0>
+ ret <2 x bfloat> %4
+}
+
+define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
+; CHECK-LABEL: fma_bf16x2_expanded_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_expanded_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_expanded_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<5>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<19>;
+; CHECK-SM70-NEXT: .reg .b32 %r<31>;
+; CHECK-SM70-NEXT: .reg .f32 %f<11>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs10}, %r15; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r27; }
+; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %f9, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f10, %r29;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %f10, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs17, %rs15, 0x0000, %p4;
+; CHECK-SM70-NEXT: selp.b16 %rs18, %rs10, 0x0000, %p3;
+; CHECK-SM70-NEXT: mov.b32 %r30, {%rs18, %rs17};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul <2 x bfloat> %a, %b
+ %2 = fadd <2 x bfloat> %1, %c
+ %3 = fcmp ogt <2 x bfloat> %2, <bfloat 0.0, bfloat 0.0>
+ %4 = select <2 x i1> %3, <2 x bfloat> %2, <2 x bfloat> <bfloat 0.0, bfloat 0.0>
+ ret <2 x bfloat> %4
+}
+
+; FMA relu shouldn't be selected if the FMA operation has multiple uses
+define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
+; CHECK-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<15>;
+; CHECK-NEXT: .reg .b32 %r<20>;
+; CHECK-NEXT: .reg .f32 %f<11>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: mov.b32 %r5, 0;
+; CHECK-NEXT: max.bf16x2 %r6, %r4, %r5;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-NEXT: cvt.u32.u16 %r7, %rs1;
+; CHECK-NEXT: shl.b32 %r8, %r7, 16;
+; CHECK-NEXT: mov.b32 %f1, %r8;
+; CHECK-NEXT: add.f32 %f2, %f1, 0f40E00000;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
+; CHECK-NEXT: cvt.u32.u16 %r9, %rs2;
+; CHECK-NEXT: shl.b32 %r10, %r9, 16;
+; CHECK-NEXT: mov.b32 %f3, %r10;
+; CHECK-NEXT: add.f32 %f4, %f3, 0f40E00000;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs6, %f4;
+; CHECK-NEXT: mov.b32 {%rs7, %rs8}, %r6;
+; CHECK-NEXT: cvt.u32.u16 %r11, %rs8;
+; CHECK-NEXT: shl.b32 %r12, %r11, 16;
+; CHECK-NEXT: mov.b32 %f5, %r12;
+; CHECK-NEXT: cvt.u32.u16 %r13, %rs6;
+; CHECK-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-NEXT: mov.b32 %f6, %r14;
+; CHECK-NEXT: add.f32 %f7, %f5, %f6;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs11, %f7;
+; CHECK-NEXT: cvt.u32.u16 %r15, %rs7;
+; CHECK-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-NEXT: mov.b32 %f8, %r16;
+; CHECK-NEXT: cvt.u32.u16 %r17, %rs4;
+; CHECK-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-NEXT: mov.b32 %f9, %r18;
+; CHECK-NEXT: add.f32 %f10, %f8, %f9;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs14, %f10;
+; CHECK-NEXT: mov.b32 %r19, {%rs14, %rs11};
+; CHECK-NEXT: st.param.b32 [func_retval0], %r19;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<15>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<20>;
+; CHECK-FTZ-NEXT: .reg .f32 %f<11>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
+; CHECK-FTZ-NEXT: max.bf16x2 %r6, %r4, %r5;
+; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r7, %rs1;
+; CHECK-FTZ-NEXT: shl.b32 %r8, %r7, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f1, %r8;
+; CHECK-FTZ-NEXT: add.ftz.f32 %f2, %f1, 0f40E00000;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r9, %rs2;
+; CHECK-FTZ-NEXT: shl.b32 %r10, %r9, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f3, %r10;
+; CHECK-FTZ-NEXT: add.ftz.f32 %f4, %f3, 0f40E00000;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f4;
+; CHECK-FTZ-NEXT: mov.b32 {%rs7, %rs8}, %r6;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r11, %rs8;
+; CHECK-FTZ-NEXT: shl.b32 %r12, %r11, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f5, %r12;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r13, %rs6;
+; CHECK-FTZ-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f6, %r14;
+; CHECK-FTZ-NEXT: add.ftz.f32 %f7, %f5, %f6;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs11, %f7;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r15, %rs7;
+; CHECK-FTZ-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f8, %r16;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r17, %rs4;
+; CHECK-FTZ-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f9, %r18;
+; CHECK-FTZ-NEXT: add.ftz.f32 %f10, %f8, %f9;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs14, %f10;
+; CHECK-FTZ-NEXT: mov.b32 %r19, {%rs14, %rs11};
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r19;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<9>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<25>;
+; CHECK-SM70-NEXT: .reg .b32 %r<61>;
+; CHECK-SM70-NEXT: .reg .f32 %f<19>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs10}, %r15; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r27; }
+; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %f9, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f10, %r29;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %f10, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs17, %rs15, 0x0000, %p4;
+; CHECK-SM70-NEXT: selp.b16 %rs18, %rs10, 0x0000, %p3;
+; CHECK-SM70-NEXT: add.f32 %f11, %f10, 0f40E00000;
+; CHECK-SM70-NEXT: mov.b32 %r30, %f11;
+; CHECK-SM70-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, %r30;
+; CHECK-SM70-NEXT: add.s32 %r33, %r32, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p5, %f11, %f11;
+; CHECK-SM70-NEXT: or.b32 %r34, %r30, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r35, %r34, %r33, %p5;
+; CHECK-SM70-NEXT: add.f32 %f12, %f9, 0f40E00000;
+; CHECK-SM70-NEXT: mov.b32 %r36, %f12;
+; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p6, %f12, %f12;
+; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p6;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r42, %rs18;
+; CHECK-SM70-NEXT: shl.b32 %r43, %r42, 16;
+; CHECK-SM70-NEXT: mov.b32 %f13, %r43;
+; CHECK-SM70-NEXT: and.b32 %r44, %r41, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f14, %r44;
+; CHECK-SM70-NEXT: add.f32 %f15, %f13, %f14;
+; CHECK-SM70-NEXT: mov.b32 %r45, %f15;
+; CHECK-SM70-NEXT: bfe.u32 %r46, %r45, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r47, %r46, %r45;
+; CHECK-SM70-NEXT: add.s32 %r48, %r47, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p7, %f15, %f15;
+; CHECK-SM70-NEXT: or.b32 %r49, %r45, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r50, %r49, %r48, %p7;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs20}, %r50; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r51, %rs17;
+; CHECK-SM70-NEXT: shl.b32 %r52, %r51, 16;
+; CHECK-SM70-NEXT: mov.b32 %f16, %r52;
+; CHECK-SM70-NEXT: and.b32 %r53, %r35, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f17, %r53;
+; CHECK-SM70-NEXT: add.f32 %f18, %f16, %f17;
+; CHECK-SM70-NEXT: mov.b32 %r54, %f18;
+; CHECK-SM70-NEXT: bfe.u32 %r55, %r54, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r56, %r55, %r54;
+; CHECK-SM70-NEXT: add.s32 %r57, %r56, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p8, %f18, %f18;
+; CHECK-SM70-NEXT: or.b32 %r58, %r54, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r59, %r58, %r57, %p8;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs23}, %r59; }
+; CHECK-SM70-NEXT: mov.b32 %r60, {%rs23, %rs20};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r60;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul <2 x bfloat> %a, %b
+ %2 = fadd <2 x bfloat> %1, %c
+ %3 = fcmp ogt <2 x bfloat> %2, <bfloat 0.0, bfloat 0.0>
+ %4 = select <2 x i1> %3, <2 x bfloat> %2, <2 x bfloat> <bfloat 0.0, bfloat 0.0>
+ %5 = fadd <2 x bfloat> %2, <bfloat 7.0, bfloat 7.0>
+ %6 = fadd <2 x bfloat> %4, %5
+ ret <2 x bfloat> %6
+}
+
+define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
+; CHECK-LABEL: fma_bf16x2_expanded_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_expanded_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_expanded_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<5>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<17>;
+; CHECK-SM70-NEXT: .reg .b32 %r<43>;
+; CHECK-SM70-NEXT: .reg .f32 %f<13>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: and.b32 %r28, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: max.f32 %f10, %f9, 0f00000000;
+; CHECK-SM70-NEXT: mov.b32 %r29, %f10;
+; CHECK-SM70-NEXT: bfe.u32 %r30, %r29, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r29;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %f10, %f10;
+; CHECK-SM70-NEXT: or.b32 %r33, %r29, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs13}, %r34; }
+; CHECK-SM70-NEXT: and.b32 %r35, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f11, %r35;
+; CHECK-SM70-NEXT: max.f32 %f12, %f11, 0f00000000;
+; CHECK-SM70-NEXT: mov.b32 %r36, %f12;
+; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %f12, %f12;
+; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p4;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r41; }
+; CHECK-SM70-NEXT: mov.b32 %r42, {%rs15, %rs13};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r42;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul <2 x bfloat> %a, %b
+ %2 = fadd <2 x bfloat> %1, %c
+ %3 = call <2 x bfloat> @llvm.maxnum.bf16x2(<2 x bfloat> %2, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
+ ret <2 x bfloat> %3
+}
+
+attributes #0 = { "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "unsafe-fp-math"="true" }
+attributes #1 = { "unsafe-fp-math"="true" }
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
new file mode 100644
index 00000000000000..16219aa9da0950
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-fma-intrinsic.ll
@@ -0,0 +1,920 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_80 | %ptxas-verify -arch=sm_80 %}
+
+; Using FTZ should emit fma.ftz.relu for f16, not for bf16
+; RUN: llc < %s -denormal-fp-math-f32=preserve-sign -march=nvptx64 -mcpu=sm_80 | FileCheck %s --check-prefixes=CHECK-FTZ
+; RUN: %if ptxas %{ llc < %s -denormal-fp-math-f32=preserve-sign -march=nvptx64 -mcpu=sm_80 | %ptxas-verify -arch=sm_80 %}
+
+; SM < 80 or (which needs PTX version >= 70) should not emit fma{.ftz}.relu
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 | FileCheck %s --check-prefixes=CHECK-SM70
+
+define half @fma_f16_no_nans(half %a, half %b, half %c) #0 {
+; CHECK-LABEL: fma_f16_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<2>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs5;
+; CHECK-SM70-NEXT: selp.b16 %rs6, %rs4, 0x0000, %p1;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs6;
+; CHECK-SM70-NEXT: ret;
+ %1 = call half @llvm.fma.f16(half %a, half %b, half %c)
+ %2 = fcmp ogt half %1, 0.0
+ %3 = select i1 %2, half %1, half 0.0
+ ret half %3
+}
+
+; FMA relu shouldn't be selected if the FMA operation has multiple uses
+define half @fma_f16_no_nans_multiple_uses_of_fma(half %a, half %b, half %c) #0 {
+; CHECK-LABEL: fma_f16_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: mov.b16 %rs5, 0x4700;
+; CHECK-NEXT: add.f16 %rs6, %rs4, %rs5;
+; CHECK-NEXT: add.f16 %rs7, %rs6, %rs4;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs7;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<8>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: mov.b16 %rs5, 0x4700;
+; CHECK-FTZ-NEXT: add.ftz.f16 %rs6, %rs4, %rs5;
+; CHECK-FTZ-NEXT: add.ftz.f16 %rs7, %rs6, %rs4;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs7;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .b16 %rs<8>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: mov.b16 %rs5, 0x4700;
+; CHECK-SM70-NEXT: add.f16 %rs6, %rs4, %rs5;
+; CHECK-SM70-NEXT: add.f16 %rs7, %rs6, %rs4;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs7;
+; CHECK-SM70-NEXT: ret;
+ %1 = call half @llvm.fma.f16(half %a, half %b, half %c)
+ %2 = fcmp ogt half %1, 0.0
+ %3 = select i1 %2, half %1, half 0.0
+ %4 = fadd half %1, 7.0
+ %5 = fadd half %4, %1
+ ret half %5
+}
+
+define half @fma_f16_maxnum_no_nans(half %a, half %b, half %c) #0 {
+; CHECK-LABEL: fma_f16_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_maxnum_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .b16 %rs<6>;
+; CHECK-SM70-NEXT: .reg .f32 %f<3>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: cvt.f32.f16 %f1, %rs4;
+; CHECK-SM70-NEXT: max.f32 %f2, %f1, 0f00000000;
+; CHECK-SM70-NEXT: cvt.rn.f16.f32 %rs5, %f2;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs5;
+; CHECK-SM70-NEXT: ret;
+ %1 = call half @llvm.fma.f16(half %a, half %b, half %c)
+ %2 = call half @llvm.maxnum.f16(half %1, half 0.0)
+ ret half %2
+}
+
+define bfloat @fma_bf16_no_nans(bfloat %a, bfloat %b, bfloat %c) #0 {
+; CHECK-LABEL: fma_bf16_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<4>;
+; CHECK-SM70-NEXT: .reg .b32 %r<14>;
+; CHECK-SM70-NEXT: .reg .f32 %f<6>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_no_nans_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_no_nans_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_no_nans_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %f5, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs1, 0x0000, %p2;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-SM70-NEXT: ret;
+ %1 = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
+ %2 = fcmp ogt bfloat %1, 0.0
+ %3 = select i1 %2, bfloat %1, bfloat 0.0
+ ret bfloat %3
+}
+
+; FMA_relu shouldn't be selected if the FMA operation has multiple uses
+define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloat %c) #0 {
+; CHECK-LABEL: fma_bf16_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<9>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .f32 %f<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs4;
+; CHECK-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-NEXT: mov.b32 %f1, %r2;
+; CHECK-NEXT: add.f32 %f2, %f1, 0f40E00000;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs6, %f2;
+; CHECK-NEXT: cvt.u32.u16 %r3, %rs6;
+; CHECK-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-NEXT: mov.b32 %f3, %r4;
+; CHECK-NEXT: add.f32 %f4, %f3, %f1;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs8, %f4;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs8;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<9>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT: .reg .f32 %f<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r1, %rs4;
+; CHECK-FTZ-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f1, %r2;
+; CHECK-FTZ-NEXT: add.ftz.f32 %f2, %f1, 0f40E00000;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f2;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r3, %rs6;
+; CHECK-FTZ-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f3, %r4;
+; CHECK-FTZ-NEXT: add.ftz.f32 %f4, %f3, %f1;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs8, %f4;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs8;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<4>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
+; CHECK-SM70-NEXT: .reg .b32 %r<27>;
+; CHECK-SM70-NEXT: .reg .f32 %f<9>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: add.f32 %f6, %f5, 0f40E00000;
+; CHECK-SM70-NEXT: mov.b32 %r14, %f6;
+; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
+; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f6, %f6;
+; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p2;
+; CHECK-SM70-NEXT: and.b32 %r20, %r19, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r20;
+; CHECK-SM70-NEXT: add.f32 %f8, %f7, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r21, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r22, %r21, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r21;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r25, %r21, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p3;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r26; }
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-SM70-NEXT: ret;
+ %1 = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
+ %2 = fcmp ogt bfloat %1, 0.0
+ %3 = select i1 %2, bfloat %1, bfloat 0.0
+ %4 = fadd bfloat %1, 7.0
+ %5 = fadd bfloat %4, %1
+ ret bfloat %5
+}
+
+define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) #0 {
+; CHECK-LABEL: fma_bf16_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_maxnum_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
+; CHECK-SM70-NEXT: .reg .b32 %r<20>;
+; CHECK-SM70-NEXT: .reg .f32 %f<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: max.f32 %f6, %f5, 0f00000000;
+; CHECK-SM70-NEXT: mov.b32 %r14, %f6;
+; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
+; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f6, %f6;
+; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r19; }
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-SM70-NEXT: ret;
+ %1 = call bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
+ %2 = call bfloat @llvm.maxnum.bf16(bfloat %1, bfloat 0.0)
+ ret bfloat %2
+}
+
+define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
+; CHECK-LABEL: fma_f16x2_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
+; CHECK-SM70-NEXT: .reg .b32 %r<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 %r5, 0;
+; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs2, 0x0000, %p2;
+; CHECK-SM70-NEXT: selp.b16 %rs4, %rs1, 0x0000, %p1;
+; CHECK-SM70-NEXT: mov.b32 %r6, {%rs4, %rs3};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT: ret;
+ %1 = call <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
+ %2 = fcmp ogt <2 x half> %1, <half 0.0, half 0.0>
+ %3 = select <2 x i1> %2, <2 x half> %1, <2 x half> <half 0.0, half 0.0>
+ ret <2 x half> %3
+}
+
+; FMA relu shouldn't be selected if the FMA operation has multiple uses
+define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
+; CHECK-LABEL: fma_f16x2_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: mov.b32 %r5, 1191200512;
+; CHECK-NEXT: add.f16x2 %r6, %r4, %r5;
+; CHECK-NEXT: add.f16x2 %r7, %r6, %r4;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r7;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<8>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: mov.b32 %r5, 1191200512;
+; CHECK-FTZ-NEXT: add.ftz.f16x2 %r6, %r4, %r5;
+; CHECK-FTZ-NEXT: add.ftz.f16x2 %r7, %r6, %r4;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r7;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .b32 %r<8>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 %r5, 1191200512;
+; CHECK-SM70-NEXT: add.f16x2 %r6, %r4, %r5;
+; CHECK-SM70-NEXT: add.f16x2 %r7, %r6, %r4;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r7;
+; CHECK-SM70-NEXT: ret;
+ %1 = call <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
+ %2 = fcmp ogt <2 x half> %1, <half 0.0, half 0.0>
+ %3 = select <2 x i1> %2, <2 x half> %1, <2 x half> <half 0.0, half 0.0>
+ %4 = fadd <2 x half> %1, <half 7.0, half 7.0>
+ %5 = fadd <2 x half> %4, %1
+ ret <2 x half> %5
+}
+
+define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) #0 {
+; CHECK-LABEL: fma_f16x2_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
+; CHECK-SM70-NEXT: .reg .b32 %r<6>;
+; CHECK-SM70-NEXT: .reg .f32 %f<5>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM70-NEXT: cvt.f32.f16 %f1, %rs2;
+; CHECK-SM70-NEXT: max.f32 %f2, %f1, 0f00000000;
+; CHECK-SM70-NEXT: cvt.rn.f16.f32 %rs3, %f2;
+; CHECK-SM70-NEXT: cvt.f32.f16 %f3, %rs1;
+; CHECK-SM70-NEXT: max.f32 %f4, %f3, 0f00000000;
+; CHECK-SM70-NEXT: cvt.rn.f16.f32 %rs4, %f4;
+; CHECK-SM70-NEXT: mov.b32 %r5, {%rs4, %rs3};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r5;
+; CHECK-SM70-NEXT: ret;
+ %1 = call <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
+ %2 = call <2 x half> @llvm.maxnum.f16x2(<2 x half> %1, <2 x half> <half 0.0, half 0.0>)
+ ret <2 x half> %2
+}
+
+define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
+; CHECK-LABEL: fma_bf16x2_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<5>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<19>;
+; CHECK-SM70-NEXT: .reg .b32 %r<31>;
+; CHECK-SM70-NEXT: .reg .f32 %f<11>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs10}, %r15; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r27; }
+; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %f9, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f10, %r29;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %f10, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs17, %rs15, 0x0000, %p4;
+; CHECK-SM70-NEXT: selp.b16 %rs18, %rs10, 0x0000, %p3;
+; CHECK-SM70-NEXT: mov.b32 %r30, {%rs18, %rs17};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT: ret;
+ %1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
+ %2 = fcmp ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
+ %3 = select <2 x i1> %2, <2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>
+ ret <2 x bfloat> %3
+}
+
+; FMA_relu shouldn't be selected if the FMA operation has multiple uses
+define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
+; CHECK-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<11>;
+; CHECK-NEXT: .reg .b32 %r<14>;
+; CHECK-NEXT: .reg .f32 %f<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-NEXT: mov.b32 %f1, %r6;
+; CHECK-NEXT: add.f32 %f2, %f1, 0f40E00000;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
+; CHECK-NEXT: cvt.u32.u16 %r7, %rs2;
+; CHECK-NEXT: shl.b32 %r8, %r7, 16;
+; CHECK-NEXT: mov.b32 %f3, %r8;
+; CHECK-NEXT: add.f32 %f4, %f3, 0f40E00000;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs6, %f4;
+; CHECK-NEXT: cvt.u32.u16 %r9, %rs6;
+; CHECK-NEXT: shl.b32 %r10, %r9, 16;
+; CHECK-NEXT: mov.b32 %f5, %r10;
+; CHECK-NEXT: add.f32 %f6, %f5, %f3;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs8, %f6;
+; CHECK-NEXT: cvt.u32.u16 %r11, %rs4;
+; CHECK-NEXT: shl.b32 %r12, %r11, 16;
+; CHECK-NEXT: mov.b32 %f7, %r12;
+; CHECK-NEXT: add.f32 %f8, %f7, %f1;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs10, %f8;
+; CHECK-NEXT: mov.b32 %r13, {%rs10, %rs8};
+; CHECK-NEXT: st.param.b32 [func_retval0], %r13;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<11>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<14>;
+; CHECK-FTZ-NEXT: .reg .f32 %f<9>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-FTZ-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f1, %r6;
+; CHECK-FTZ-NEXT: add.ftz.f32 %f2, %f1, 0f40E00000;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r7, %rs2;
+; CHECK-FTZ-NEXT: shl.b32 %r8, %r7, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f3, %r8;
+; CHECK-FTZ-NEXT: add.ftz.f32 %f4, %f3, 0f40E00000;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f4;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r9, %rs6;
+; CHECK-FTZ-NEXT: shl.b32 %r10, %r9, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f5, %r10;
+; CHECK-FTZ-NEXT: add.ftz.f32 %f6, %f5, %f3;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs8, %f6;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r11, %rs4;
+; CHECK-FTZ-NEXT: shl.b32 %r12, %r11, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f7, %r12;
+; CHECK-FTZ-NEXT: add.ftz.f32 %f8, %f7, %f1;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs10, %f8;
+; CHECK-FTZ-NEXT: mov.b32 %r13, {%rs10, %rs8};
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r13;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<7>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<17>;
+; CHECK-SM70-NEXT: .reg .b32 %r<57>;
+; CHECK-SM70-NEXT: .reg .f32 %f<17>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: and.b32 %r28, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: add.f32 %f10, %f9, 0f40E00000;
+; CHECK-SM70-NEXT: mov.b32 %r29, %f10;
+; CHECK-SM70-NEXT: bfe.u32 %r30, %r29, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r29;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %f10, %f10;
+; CHECK-SM70-NEXT: or.b32 %r33, %r29, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
+; CHECK-SM70-NEXT: and.b32 %r35, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f11, %r35;
+; CHECK-SM70-NEXT: add.f32 %f12, %f11, 0f40E00000;
+; CHECK-SM70-NEXT: mov.b32 %r36, %f12;
+; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %f12, %f12;
+; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p4;
+; CHECK-SM70-NEXT: and.b32 %r42, %r41, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f13, %r42;
+; CHECK-SM70-NEXT: add.f32 %f14, %f13, %f11;
+; CHECK-SM70-NEXT: mov.b32 %r43, %f14;
+; CHECK-SM70-NEXT: bfe.u32 %r44, %r43, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r45, %r44, %r43;
+; CHECK-SM70-NEXT: add.s32 %r46, %r45, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p5, %f14, %f14;
+; CHECK-SM70-NEXT: or.b32 %r47, %r43, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r48, %r47, %r46, %p5;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs13}, %r48; }
+; CHECK-SM70-NEXT: and.b32 %r49, %r34, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f15, %r49;
+; CHECK-SM70-NEXT: add.f32 %f16, %f15, %f9;
+; CHECK-SM70-NEXT: mov.b32 %r50, %f16;
+; CHECK-SM70-NEXT: bfe.u32 %r51, %r50, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r52, %r51, %r50;
+; CHECK-SM70-NEXT: add.s32 %r53, %r52, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p6, %f16, %f16;
+; CHECK-SM70-NEXT: or.b32 %r54, %r50, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r55, %r54, %r53, %p6;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r55; }
+; CHECK-SM70-NEXT: mov.b32 %r56, {%rs15, %rs13};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r56;
+; CHECK-SM70-NEXT: ret;
+ %1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
+ %2 = fcmp ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
+ %3 = select <2 x i1> %2, <2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>
+ %4 = fadd <2 x bfloat> %1, <bfloat 7.0, bfloat 7.0>
+ %5 = fadd <2 x bfloat> %4, %1
+ ret <2 x bfloat> %5
+}
+
+define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) #0 {
+; CHECK-LABEL: fma_bf16x2_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<5>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<17>;
+; CHECK-SM70-NEXT: .reg .b32 %r<43>;
+; CHECK-SM70-NEXT: .reg .f32 %f<13>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: and.b32 %r28, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: max.f32 %f10, %f9, 0f00000000;
+; CHECK-SM70-NEXT: mov.b32 %r29, %f10;
+; CHECK-SM70-NEXT: bfe.u32 %r30, %r29, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r29;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %f10, %f10;
+; CHECK-SM70-NEXT: or.b32 %r33, %r29, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs13}, %r34; }
+; CHECK-SM70-NEXT: and.b32 %r35, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f11, %r35;
+; CHECK-SM70-NEXT: max.f32 %f12, %f11, 0f00000000;
+; CHECK-SM70-NEXT: mov.b32 %r36, %f12;
+; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %f12, %f12;
+; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p4;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r41; }
+; CHECK-SM70-NEXT: mov.b32 %r42, {%rs15, %rs13};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r42;
+; CHECK-SM70-NEXT: ret;
+ %1 = call <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
+ %2 = call <2 x bfloat> @llvm.maxnum.bf16x2(<2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
+ ret <2 x bfloat> %2
+}
+
+attributes #0 = { "no-nans-fp-math"="true" "no-signed-zeros-fp-math"="true" "unsafe-fp-math"="true" }
+attributes #1 = { "unsafe-fp-math"="true" }
diff --git a/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll b/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
new file mode 100644
index 00000000000000..af21bada7783be
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/fma-relu-instruction-flag.ll
@@ -0,0 +1,1912 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 | FileCheck %s
+; RUN: %if ptxas %{ llc < %s -march=nvptx64 -mcpu=sm_80 | %ptxas-verify -arch=sm_80 %}
+
+; Using FTZ should emit fma.ftz.relu for f16, not for bf16
+; RUN: llc < %s -denormal-fp-math-f32=preserve-sign -march=nvptx64 -mcpu=sm_80 | FileCheck %s --check-prefixes=CHECK-FTZ
+; RUN: %if ptxas %{ llc < %s -denormal-fp-math-f32=preserve-sign -march=nvptx64 -mcpu=sm_80 | %ptxas-verify -arch=sm_80 %}
+
+; SM < 80 or (which needs PTX version >= 70) should not emit fma{.ftz}.relu
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 | FileCheck %s --check-prefixes=CHECK-SM70
+
+define half @fma_f16_expanded_no_nans(half %a, half %b, half %c) {
+; CHECK-LABEL: fma_f16_expanded_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_expanded_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_expanded_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<2>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs5;
+; CHECK-SM70-NEXT: selp.b16 %rs6, %rs4, 0x0000, %p1;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs6;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul fast half %a, %b
+ %2 = fadd fast half %1, %c
+ %3 = fcmp nsz ogt half %2, 0.0
+ %4 = select nsz i1 %3, half %2, half 0.0
+ ret half %4
+}
+
+; FMA relu shouldn't be selected if the FMA operation has multiple uses
+define half @fma_f16_expanded_no_nans_multiple_uses_of_fma(half %a, half %b, half %c) {
+; CHECK-LABEL: fma_f16_expanded_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-NEXT: max.f16 %rs6, %rs4, %rs5;
+; CHECK-NEXT: mov.b16 %rs7, 0x4700;
+; CHECK-NEXT: add.rn.f16 %rs8, %rs4, %rs7;
+; CHECK-NEXT: add.rn.f16 %rs9, %rs6, %rs8;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs9;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<10>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-FTZ-NEXT: max.ftz.f16 %rs6, %rs4, %rs5;
+; CHECK-FTZ-NEXT: mov.b16 %rs7, 0x4700;
+; CHECK-FTZ-NEXT: add.rn.ftz.f16 %rs8, %rs4, %rs7;
+; CHECK-FTZ-NEXT: add.rn.ftz.f16 %rs9, %rs6, %rs8;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs9;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<2>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<10>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs5;
+; CHECK-SM70-NEXT: selp.b16 %rs6, %rs4, 0x0000, %p1;
+; CHECK-SM70-NEXT: mov.b16 %rs7, 0x4700;
+; CHECK-SM70-NEXT: add.rn.f16 %rs8, %rs4, %rs7;
+; CHECK-SM70-NEXT: add.rn.f16 %rs9, %rs6, %rs8;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs9;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul fast half %a, %b
+ %2 = fadd fast half %1, %c
+ %3 = fcmp nsz ogt half %2, 0.0
+ %4 = select nsz i1 %3, half %2, half 0.0
+ %5 = fadd half %2, 7.0
+ %6 = fadd half %4, %5
+ ret half %6
+}
+
+define half @fma_f16_expanded_maxnum_no_nans(half %a, half %b, half %c) {
+; CHECK-LABEL: fma_f16_expanded_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_maxnum_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_expanded_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_expanded_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .b16 %rs<6>;
+; CHECK-SM70-NEXT: .reg .f32 %f<3>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_expanded_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_expanded_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: cvt.f32.f16 %f1, %rs4;
+; CHECK-SM70-NEXT: max.f32 %f2, %f1, 0f00000000;
+; CHECK-SM70-NEXT: cvt.rn.f16.f32 %rs5, %f2;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs5;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul fast half %a, %b
+ %2 = fadd fast half %1, %c
+ %3 = call nsz half @llvm.maxnum.f16(half %2, half 0.0)
+ ret half %3
+}
+
+define bfloat @fma_bf16_expanded_no_nans(bfloat %a, bfloat %b, bfloat %c) {
+; CHECK-LABEL: fma_bf16_expanded_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_expanded_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_expanded_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<4>;
+; CHECK-SM70-NEXT: .reg .b32 %r<14>;
+; CHECK-SM70-NEXT: .reg .f32 %f<6>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_expanded_no_nans_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_expanded_no_nans_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %f5, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs1, 0x0000, %p2;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul fast bfloat %a, %b
+ %2 = fadd fast bfloat %1, %c
+ %3 = fcmp nsz ogt bfloat %2, 0.0
+ %4 = select nsz i1 %3, bfloat %2, bfloat 0.0
+ ret bfloat %4
+}
+
+; FMA relu shouldn't be selected if the FMA operation has multiple uses
+define bfloat @fma_bf16_expanded_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloat %c) {
+; CHECK-LABEL: fma_bf16_expanded_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<12>;
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-NEXT: .reg .f32 %f<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-NEXT: max.bf16 %rs6, %rs4, %rs5;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs4;
+; CHECK-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-NEXT: mov.b32 %f1, %r2;
+; CHECK-NEXT: add.rn.f32 %f2, %f1, 0f40E00000;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs8, %f2;
+; CHECK-NEXT: cvt.u32.u16 %r3, %rs6;
+; CHECK-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-NEXT: mov.b32 %f3, %r4;
+; CHECK-NEXT: cvt.u32.u16 %r5, %rs8;
+; CHECK-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-NEXT: mov.b32 %f4, %r6;
+; CHECK-NEXT: add.rn.f32 %f5, %f3, %f4;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs11, %f5;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs11;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<12>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<7>;
+; CHECK-FTZ-NEXT: .reg .f32 %f<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-FTZ-NEXT: max.bf16 %rs6, %rs4, %rs5;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r1, %rs4;
+; CHECK-FTZ-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f1, %r2;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f2, %f1, 0f40E00000;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs8, %f2;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r3, %rs6;
+; CHECK-FTZ-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f3, %r4;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r5, %rs8;
+; CHECK-FTZ-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f4, %r6;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f5, %f3, %f4;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs11, %f5;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs11;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<5>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
+; CHECK-SM70-NEXT: .reg .b32 %r<29>;
+; CHECK-SM70-NEXT: .reg .f32 %f<10>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %f5, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs1, 0x0000, %p2;
+; CHECK-SM70-NEXT: add.rn.f32 %f6, %f5, 0f40E00000;
+; CHECK-SM70-NEXT: mov.b32 %r14, %f6;
+; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
+; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %f6, %f6;
+; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs3;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: and.b32 %r22, %r19, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f8, %r22;
+; CHECK-SM70-NEXT: add.rn.f32 %f9, %f7, %f8;
+; CHECK-SM70-NEXT: mov.b32 %r23, %f9;
+; CHECK-SM70-NEXT: bfe.u32 %r24, %r23, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, %r23;
+; CHECK-SM70-NEXT: add.s32 %r26, %r25, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %f9, %f9;
+; CHECK-SM70-NEXT: or.b32 %r27, %r23, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r28, %r27, %r26, %p4;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs5}, %r28; }
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs5;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul fast bfloat %a, %b
+ %2 = fadd fast bfloat %1, %c
+ %3 = fcmp nsz ogt bfloat %2, 0.0
+ %4 = select nsz i1 %3, bfloat %2, bfloat 0.0
+ %5 = fadd bfloat %2, 7.0
+ %6 = fadd bfloat %4, %5
+ ret bfloat %6
+}
+
+define bfloat @fma_bf16_expanded_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) {
+;
+;
+; CHECK-LABEL: fma_bf16_expanded_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_maxnum_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_expanded_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_expanded_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_expanded_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_expanded_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
+; CHECK-SM70-NEXT: .reg .b32 %r<20>;
+; CHECK-SM70-NEXT: .reg .f32 %f<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_expanded_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_expanded_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: max.f32 %f6, %f5, 0f00000000;
+; CHECK-SM70-NEXT: mov.b32 %r14, %f6;
+; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
+; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f6, %f6;
+; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r19; }
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul fast bfloat %a, %b
+ %2 = fadd fast bfloat %1, %c
+ %3 = call nsz bfloat @llvm.maxnum.bf16(bfloat %2, bfloat 0.0)
+ ret bfloat %3
+}
+
+define <2 x half> @fma_f16x2_expanded_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
+;
+;
+; CHECK-LABEL: fma_f16x2_expanded_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_expanded_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_expanded_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
+; CHECK-SM70-NEXT: .reg .b32 %r<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 %r5, 0;
+; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs2, 0x0000, %p2;
+; CHECK-SM70-NEXT: selp.b16 %rs4, %rs1, 0x0000, %p1;
+; CHECK-SM70-NEXT: mov.b32 %r6, {%rs4, %rs3};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul fast <2 x half> %a, %b
+ %2 = fadd fast <2 x half> %1, %c
+ %3 = fcmp nsz ogt <2 x half> %2, <half 0.0, half 0.0>
+ %4 = select nsz <2 x i1> %3, <2 x half> %2, <2 x half> <half 0.0, half 0.0>
+ ret <2 x half> %4
+}
+
+; FMA relu shouldn't be selected if the FMA operation has multiple uses
+define <2 x half> @fma_f16x2_expanded_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
+;
+;
+; CHECK-LABEL: fma_f16x2_expanded_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<10>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: mov.b32 %r5, 0;
+; CHECK-NEXT: max.f16x2 %r6, %r4, %r5;
+; CHECK-NEXT: mov.b32 %r7, 1191200512;
+; CHECK-NEXT: add.rn.f16x2 %r8, %r4, %r7;
+; CHECK-NEXT: add.rn.f16x2 %r9, %r6, %r8;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<10>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
+; CHECK-FTZ-NEXT: max.ftz.f16x2 %r6, %r4, %r5;
+; CHECK-FTZ-NEXT: mov.b32 %r7, 1191200512;
+; CHECK-FTZ-NEXT: add.rn.ftz.f16x2 %r8, %r4, %r7;
+; CHECK-FTZ-NEXT: add.rn.ftz.f16x2 %r9, %r6, %r8;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
+; CHECK-SM70-NEXT: .reg .b32 %r<10>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 %r5, 0;
+; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs2, 0x0000, %p2;
+; CHECK-SM70-NEXT: selp.b16 %rs4, %rs1, 0x0000, %p1;
+; CHECK-SM70-NEXT: mov.b32 %r6, {%rs4, %rs3};
+; CHECK-SM70-NEXT: mov.b32 %r7, 1191200512;
+; CHECK-SM70-NEXT: add.rn.f16x2 %r8, %r4, %r7;
+; CHECK-SM70-NEXT: add.rn.f16x2 %r9, %r6, %r8;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r9;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul fast <2 x half> %a, %b
+ %2 = fadd fast <2 x half> %1, %c
+ %3 = fcmp nsz ogt <2 x half> %2, <half 0.0, half 0.0>
+ %4 = select nsz <2 x i1> %3, <2 x half> %2, <2 x half> <half 0.0, half 0.0>
+ %5 = fadd <2 x half> %2, <half 7.0, half 7.0>
+ %6 = fadd <2 x half> %4, %5
+ ret <2 x half> %6
+}
+
+define <2 x half> @fma_f16x2_expanded_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
+; CHECK-LABEL: fma_f16x2_expanded_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_expanded_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_expanded_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
+; CHECK-SM70-NEXT: .reg .b32 %r<6>;
+; CHECK-SM70-NEXT: .reg .f32 %f<5>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM70-NEXT: cvt.f32.f16 %f1, %rs2;
+; CHECK-SM70-NEXT: max.f32 %f2, %f1, 0f00000000;
+; CHECK-SM70-NEXT: cvt.rn.f16.f32 %rs3, %f2;
+; CHECK-SM70-NEXT: cvt.f32.f16 %f3, %rs1;
+; CHECK-SM70-NEXT: max.f32 %f4, %f3, 0f00000000;
+; CHECK-SM70-NEXT: cvt.rn.f16.f32 %rs4, %f4;
+; CHECK-SM70-NEXT: mov.b32 %r5, {%rs4, %rs3};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r5;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul fast <2 x half> %a, %b
+ %2 = fadd fast <2 x half> %1, %c
+ %3 = call nsz <2 x half> @llvm.maxnum.f16x2(<2 x half> %2, <2 x half> <half 0.0, half 0.0>)
+ ret <2 x half> %3
+}
+
+define <2 x bfloat> @fma_bf16x2_expanded_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
+; CHECK-LABEL: fma_bf16x2_expanded_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_expanded_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_expanded_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<5>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<19>;
+; CHECK-SM70-NEXT: .reg .b32 %r<31>;
+; CHECK-SM70-NEXT: .reg .f32 %f<11>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs10}, %r15; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r27; }
+; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %f9, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f10, %r29;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %f10, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs17, %rs15, 0x0000, %p4;
+; CHECK-SM70-NEXT: selp.b16 %rs18, %rs10, 0x0000, %p3;
+; CHECK-SM70-NEXT: mov.b32 %r30, {%rs18, %rs17};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul fast <2 x bfloat> %a, %b
+ %2 = fadd fast <2 x bfloat> %1, %c
+ %3 = fcmp nsz ogt <2 x bfloat> %2, <bfloat 0.0, bfloat 0.0>
+ %4 = select nsz <2 x i1> %3, <2 x bfloat> %2, <2 x bfloat> <bfloat 0.0, bfloat 0.0>
+ ret <2 x bfloat> %4
+}
+
+; FMA relu shouldn't be selected if the FMA operation has multiple uses
+define <2 x bfloat> @fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
+; CHECK-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<15>;
+; CHECK-NEXT: .reg .b32 %r<20>;
+; CHECK-NEXT: .reg .f32 %f<11>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: mov.b32 %r5, 0;
+; CHECK-NEXT: max.bf16x2 %r6, %r4, %r5;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-NEXT: cvt.u32.u16 %r7, %rs1;
+; CHECK-NEXT: shl.b32 %r8, %r7, 16;
+; CHECK-NEXT: mov.b32 %f1, %r8;
+; CHECK-NEXT: add.rn.f32 %f2, %f1, 0f40E00000;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
+; CHECK-NEXT: cvt.u32.u16 %r9, %rs2;
+; CHECK-NEXT: shl.b32 %r10, %r9, 16;
+; CHECK-NEXT: mov.b32 %f3, %r10;
+; CHECK-NEXT: add.rn.f32 %f4, %f3, 0f40E00000;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs6, %f4;
+; CHECK-NEXT: mov.b32 {%rs7, %rs8}, %r6;
+; CHECK-NEXT: cvt.u32.u16 %r11, %rs8;
+; CHECK-NEXT: shl.b32 %r12, %r11, 16;
+; CHECK-NEXT: mov.b32 %f5, %r12;
+; CHECK-NEXT: cvt.u32.u16 %r13, %rs6;
+; CHECK-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-NEXT: mov.b32 %f6, %r14;
+; CHECK-NEXT: add.rn.f32 %f7, %f5, %f6;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs11, %f7;
+; CHECK-NEXT: cvt.u32.u16 %r15, %rs7;
+; CHECK-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-NEXT: mov.b32 %f8, %r16;
+; CHECK-NEXT: cvt.u32.u16 %r17, %rs4;
+; CHECK-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-NEXT: mov.b32 %f9, %r18;
+; CHECK-NEXT: add.rn.f32 %f10, %f8, %f9;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs14, %f10;
+; CHECK-NEXT: mov.b32 %r19, {%rs14, %rs11};
+; CHECK-NEXT: st.param.b32 [func_retval0], %r19;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<15>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<20>;
+; CHECK-FTZ-NEXT: .reg .f32 %f<11>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
+; CHECK-FTZ-NEXT: max.bf16x2 %r6, %r4, %r5;
+; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r7, %rs1;
+; CHECK-FTZ-NEXT: shl.b32 %r8, %r7, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f1, %r8;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f2, %f1, 0f40E00000;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r9, %rs2;
+; CHECK-FTZ-NEXT: shl.b32 %r10, %r9, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f3, %r10;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f4, %f3, 0f40E00000;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f4;
+; CHECK-FTZ-NEXT: mov.b32 {%rs7, %rs8}, %r6;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r11, %rs8;
+; CHECK-FTZ-NEXT: shl.b32 %r12, %r11, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f5, %r12;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r13, %rs6;
+; CHECK-FTZ-NEXT: shl.b32 %r14, %r13, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f6, %r14;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f7, %f5, %f6;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs11, %f7;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r15, %rs7;
+; CHECK-FTZ-NEXT: shl.b32 %r16, %r15, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f8, %r16;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r17, %rs4;
+; CHECK-FTZ-NEXT: shl.b32 %r18, %r17, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f9, %r18;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f10, %f8, %f9;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs14, %f10;
+; CHECK-FTZ-NEXT: mov.b32 %r19, {%rs14, %rs11};
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r19;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_expanded_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<9>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<25>;
+; CHECK-SM70-NEXT: .reg .b32 %r<61>;
+; CHECK-SM70-NEXT: .reg .f32 %f<19>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs10}, %r15; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r27; }
+; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %f9, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f10, %r29;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %f10, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs17, %rs15, 0x0000, %p4;
+; CHECK-SM70-NEXT: selp.b16 %rs18, %rs10, 0x0000, %p3;
+; CHECK-SM70-NEXT: add.rn.f32 %f11, %f10, 0f40E00000;
+; CHECK-SM70-NEXT: mov.b32 %r30, %f11;
+; CHECK-SM70-NEXT: bfe.u32 %r31, %r30, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, %r30;
+; CHECK-SM70-NEXT: add.s32 %r33, %r32, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p5, %f11, %f11;
+; CHECK-SM70-NEXT: or.b32 %r34, %r30, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r35, %r34, %r33, %p5;
+; CHECK-SM70-NEXT: add.rn.f32 %f12, %f9, 0f40E00000;
+; CHECK-SM70-NEXT: mov.b32 %r36, %f12;
+; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p6, %f12, %f12;
+; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p6;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r42, %rs18;
+; CHECK-SM70-NEXT: shl.b32 %r43, %r42, 16;
+; CHECK-SM70-NEXT: mov.b32 %f13, %r43;
+; CHECK-SM70-NEXT: and.b32 %r44, %r41, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f14, %r44;
+; CHECK-SM70-NEXT: add.rn.f32 %f15, %f13, %f14;
+; CHECK-SM70-NEXT: mov.b32 %r45, %f15;
+; CHECK-SM70-NEXT: bfe.u32 %r46, %r45, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r47, %r46, %r45;
+; CHECK-SM70-NEXT: add.s32 %r48, %r47, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p7, %f15, %f15;
+; CHECK-SM70-NEXT: or.b32 %r49, %r45, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r50, %r49, %r48, %p7;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs20}, %r50; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r51, %rs17;
+; CHECK-SM70-NEXT: shl.b32 %r52, %r51, 16;
+; CHECK-SM70-NEXT: mov.b32 %f16, %r52;
+; CHECK-SM70-NEXT: and.b32 %r53, %r35, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f17, %r53;
+; CHECK-SM70-NEXT: add.rn.f32 %f18, %f16, %f17;
+; CHECK-SM70-NEXT: mov.b32 %r54, %f18;
+; CHECK-SM70-NEXT: bfe.u32 %r55, %r54, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r56, %r55, %r54;
+; CHECK-SM70-NEXT: add.s32 %r57, %r56, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p8, %f18, %f18;
+; CHECK-SM70-NEXT: or.b32 %r58, %r54, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r59, %r58, %r57, %p8;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs23}, %r59; }
+; CHECK-SM70-NEXT: mov.b32 %r60, {%rs23, %rs20};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r60;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul fast <2 x bfloat> %a, %b
+ %2 = fadd fast <2 x bfloat> %1, %c
+ %3 = fcmp nsz ogt <2 x bfloat> %2, <bfloat 0.0, bfloat 0.0>
+ %4 = select nsz <2 x i1> %3, <2 x bfloat> %2, <2 x bfloat> <bfloat 0.0, bfloat 0.0>
+ %5 = fadd <2 x bfloat> %2, <bfloat 7.0, bfloat 7.0>
+ %6 = fadd <2 x bfloat> %4, %5
+ ret <2 x bfloat> %6
+}
+
+define <2 x bfloat> @fma_bf16x2_expanded_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
+; CHECK-LABEL: fma_bf16x2_expanded_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_expanded_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_expanded_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<5>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<17>;
+; CHECK-SM70-NEXT: .reg .b32 %r<43>;
+; CHECK-SM70-NEXT: .reg .f32 %f<13>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_expanded_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_expanded_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_expanded_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: and.b32 %r28, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: max.f32 %f10, %f9, 0f00000000;
+; CHECK-SM70-NEXT: mov.b32 %r29, %f10;
+; CHECK-SM70-NEXT: bfe.u32 %r30, %r29, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r29;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %f10, %f10;
+; CHECK-SM70-NEXT: or.b32 %r33, %r29, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs13}, %r34; }
+; CHECK-SM70-NEXT: and.b32 %r35, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f11, %r35;
+; CHECK-SM70-NEXT: max.f32 %f12, %f11, 0f00000000;
+; CHECK-SM70-NEXT: mov.b32 %r36, %f12;
+; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %f12, %f12;
+; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p4;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r41; }
+; CHECK-SM70-NEXT: mov.b32 %r42, {%rs15, %rs13};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r42;
+; CHECK-SM70-NEXT: ret;
+ %1 = fmul fast <2 x bfloat> %a, %b
+ %2 = fadd fast <2 x bfloat> %1, %c
+ %3 = call nsz <2 x bfloat> @llvm.maxnum.bf16x2(<2 x bfloat> %2, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
+ ret <2 x bfloat> %3
+}
+
+define half @fma_f16_no_nans(half %a, half %b, half %c) {
+; CHECK-LABEL: fma_f16_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<2>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: mov.b16 %rs5, 0x0000;
+; CHECK-SM70-NEXT: setp.gt.f16 %p1, %rs4, %rs5;
+; CHECK-SM70-NEXT: selp.b16 %rs6, %rs4, 0x0000, %p1;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs6;
+; CHECK-SM70-NEXT: ret;
+ %1 = call nnan half @llvm.fma.f16(half %a, half %b, half %c)
+ %2 = fcmp nsz ogt half %1, 0.0
+ %3 = select nsz i1 %2, half %1, half 0.0
+ ret half %3
+}
+
+; FMA relu shouldn't be selected if the FMA operation has multiple uses
+define half @fma_f16_no_nans_multiple_uses_of_fma(half %a, half %b, half %c) {
+; CHECK-LABEL: fma_f16_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: mov.b16 %rs5, 0x4700;
+; CHECK-NEXT: add.rn.f16 %rs6, %rs4, %rs5;
+; CHECK-NEXT: add.rn.f16 %rs7, %rs6, %rs4;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs7;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<8>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: mov.b16 %rs5, 0x4700;
+; CHECK-FTZ-NEXT: add.rn.ftz.f16 %rs6, %rs4, %rs5;
+; CHECK-FTZ-NEXT: add.rn.ftz.f16 %rs7, %rs6, %rs4;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs7;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .b16 %rs<8>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: mov.b16 %rs5, 0x4700;
+; CHECK-SM70-NEXT: add.rn.f16 %rs6, %rs4, %rs5;
+; CHECK-SM70-NEXT: add.rn.f16 %rs7, %rs6, %rs4;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs7;
+; CHECK-SM70-NEXT: ret;
+ %1 = call nnan half @llvm.fma.f16(half %a, half %b, half %c)
+ %2 = fcmp nsz ogt half %1, 0.0
+ %3 = select nsz i1 %2, half %1, half 0.0
+ %4 = fadd half %1, 7.0
+ %5 = fadd half %4, %1
+ ret half %5
+}
+
+define half @fma_f16_maxnum_no_nans(half %a, half %b, half %c) {
+; CHECK-LABEL: fma_f16_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_f16_maxnum_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_f16_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_f16_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_f16_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_f16_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_f16_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .b16 %rs<6>;
+; CHECK-SM70-NEXT: .reg .f32 %f<3>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b16 %rs1, [fma_f16_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b16 %rs2, [fma_f16_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b16 %rs3, [fma_f16_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: fma.rn.f16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-SM70-NEXT: cvt.f32.f16 %f1, %rs4;
+; CHECK-SM70-NEXT: max.f32 %f2, %f1, 0f00000000;
+; CHECK-SM70-NEXT: cvt.rn.f16.f32 %rs5, %f2;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs5;
+; CHECK-SM70-NEXT: ret;
+ %1 = call nnan half @llvm.fma.f16(half %a, half %b, half %c)
+ %2 = call nsz half @llvm.maxnum.f16(half %1, half 0.0)
+ ret half %2
+}
+
+define bfloat @fma_bf16_no_nans(bfloat %a, bfloat %b, bfloat %c) {
+; CHECK-LABEL: fma_bf16_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<4>;
+; CHECK-SM70-NEXT: .reg .b32 %r<14>;
+; CHECK-SM70-NEXT: .reg .f32 %f<6>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_no_nans_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_no_nans_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_no_nans_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r12; }
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: setp.gt.f32 %p2, %f5, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs1, 0x0000, %p2;
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs3;
+; CHECK-SM70-NEXT: ret;
+ %1 = call nnan bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
+ %2 = fcmp nsz ogt bfloat %1, 0.0
+ %3 = select nsz i1 %2, bfloat %1, bfloat 0.0
+ ret bfloat %3
+}
+
+; FMA_relu shouldn't be selected if the FMA operation has multiple uses
+define bfloat @fma_bf16_no_nans_multiple_uses_of_fma(bfloat %a, bfloat %b, bfloat %c) {
+; CHECK-LABEL: fma_bf16_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<9>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .f32 %f<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: fma.rn.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: cvt.u32.u16 %r1, %rs4;
+; CHECK-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-NEXT: mov.b32 %f1, %r2;
+; CHECK-NEXT: add.rn.f32 %f2, %f1, 0f40E00000;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs6, %f2;
+; CHECK-NEXT: cvt.u32.u16 %r3, %rs6;
+; CHECK-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-NEXT: mov.b32 %f3, %r4;
+; CHECK-NEXT: add.rn.f32 %f4, %f3, %f1;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs8, %f4;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs8;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<9>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT: .reg .f32 %f<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: fma.rn.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r1, %rs4;
+; CHECK-FTZ-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f1, %r2;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f2, %f1, 0f40E00000;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f2;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r3, %rs6;
+; CHECK-FTZ-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f3, %r4;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f4, %f3, %f1;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs8, %f4;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs8;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<4>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
+; CHECK-SM70-NEXT: .reg .b32 %r<27>;
+; CHECK-SM70-NEXT: .reg .f32 %f<9>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: add.rn.f32 %f6, %f5, 0f40E00000;
+; CHECK-SM70-NEXT: mov.b32 %r14, %f6;
+; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
+; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f6, %f6;
+; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p2;
+; CHECK-SM70-NEXT: and.b32 %r20, %r19, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r20;
+; CHECK-SM70-NEXT: add.rn.f32 %f8, %f7, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r21, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r22, %r21, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r23, %r22, %r21;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r25, %r21, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r26, %r25, %r24, %p3;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r26; }
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-SM70-NEXT: ret;
+ %1 = call nnan bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
+ %2 = fcmp nsz ogt bfloat %1, 0.0
+ %3 = select nsz i1 %2, bfloat %1, bfloat 0.0
+ %4 = fadd bfloat %1, 7.0
+ %5 = fadd bfloat %4, %1
+ ret bfloat %5
+}
+
+define bfloat @fma_bf16_maxnum_no_nans(bfloat %a, bfloat %b, bfloat %c) {
+; CHECK-LABEL: fma_bf16_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b16 %rs1, [fma_bf16_maxnum_no_nans_param_0];
+; CHECK-NEXT: ld.param.b16 %rs2, [fma_bf16_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b16 %rs3, [fma_bf16_maxnum_no_nans_param_2];
+; CHECK-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b16 %rs1, [fma_bf16_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs2, [fma_bf16_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b16 %rs3, [fma_bf16_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16 %rs4, %rs1, %rs2, %rs3;
+; CHECK-FTZ-NEXT: st.param.b16 [func_retval0], %rs4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<3>;
+; CHECK-SM70-NEXT: .reg .b32 %r<20>;
+; CHECK-SM70-NEXT: .reg .f32 %f<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.u16 %r1, [fma_bf16_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: shl.b32 %r2, %r1, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r2;
+; CHECK-SM70-NEXT: ld.param.u16 %r3, [fma_bf16_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: shl.b32 %r4, %r3, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r4;
+; CHECK-SM70-NEXT: ld.param.u16 %r5, [fma_bf16_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r6;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r7, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r8, %r7, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r9, %r8, %r7;
+; CHECK-SM70-NEXT: add.s32 %r10, %r9, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r11, %r7, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r12, %r11, %r10, %p1;
+; CHECK-SM70-NEXT: and.b32 %r13, %r12, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r13;
+; CHECK-SM70-NEXT: max.f32 %f6, %f5, 0f00000000;
+; CHECK-SM70-NEXT: mov.b32 %r14, %f6;
+; CHECK-SM70-NEXT: bfe.u32 %r15, %r14, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r16, %r15, %r14;
+; CHECK-SM70-NEXT: add.s32 %r17, %r16, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f6, %f6;
+; CHECK-SM70-NEXT: or.b32 %r18, %r14, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r19, %r18, %r17, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r19; }
+; CHECK-SM70-NEXT: st.param.b16 [func_retval0], %rs1;
+; CHECK-SM70-NEXT: ret;
+ %1 = call nnan bfloat @llvm.fma.bf16(bfloat %a, bfloat %b, bfloat %c)
+ %2 = call nsz bfloat @llvm.maxnum.bf16(bfloat %1, bfloat 0.0)
+ ret bfloat %2
+}
+
+define <2 x half> @fma_f16x2_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
+; CHECK-LABEL: fma_f16x2_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<7>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
+; CHECK-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: mov.b32 %r5, 0;
+; CHECK-NEXT: max.f16x2 %r6, %r4, %r5;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<7>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: mov.b32 %r5, 0;
+; CHECK-FTZ-NEXT: max.ftz.f16x2 %r6, %r4, %r5;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<3>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
+; CHECK-SM70-NEXT: .reg .b32 %r<7>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 %r5, 0;
+; CHECK-SM70-NEXT: setp.gt.f16x2 %p1|%p2, %r4, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM70-NEXT: selp.b16 %rs3, %rs2, 0x0000, %p2;
+; CHECK-SM70-NEXT: selp.b16 %rs4, %rs1, 0x0000, %p1;
+; CHECK-SM70-NEXT: mov.b32 %r6, {%rs4, %rs3};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r6;
+; CHECK-SM70-NEXT: ret;
+ %1 = call <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
+ %2 = fcmp nsz ogt <2 x half> %1, <half 0.0, half 0.0>
+ %3 = select nsz <2 x i1> %2, <2 x half> %1, <2 x half> <half 0.0, half 0.0>
+ ret <2 x half> %3
+}
+
+; FMA relu shouldn't be selected if the FMA operation has multiple uses
+define <2 x half> @fma_f16x2_no_nans_multiple_uses_of_fma(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
+; CHECK-LABEL: fma_f16x2_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<8>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: mov.b32 %r5, 1191200512;
+; CHECK-NEXT: add.rn.f16x2 %r6, %r4, %r5;
+; CHECK-NEXT: add.rn.f16x2 %r7, %r6, %r4;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r7;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<8>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: mov.b32 %r5, 1191200512;
+; CHECK-FTZ-NEXT: add.rn.ftz.f16x2 %r6, %r4, %r5;
+; CHECK-FTZ-NEXT: add.rn.ftz.f16x2 %r7, %r6, %r4;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r7;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .b32 %r<8>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 %r5, 1191200512;
+; CHECK-SM70-NEXT: add.rn.f16x2 %r6, %r4, %r5;
+; CHECK-SM70-NEXT: add.rn.f16x2 %r7, %r6, %r4;
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r7;
+; CHECK-SM70-NEXT: ret;
+ %1 = call nnan <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
+ %2 = fcmp nsz ogt <2 x half> %1, <half 0.0, half 0.0>
+ %3 = select nsz <2 x i1> %2, <2 x half> %1, <2 x half> <half 0.0, half 0.0>
+ %4 = fadd <2 x half> %1, <half 7.0, half 7.0>
+ %5 = fadd <2 x half> %4, %1
+ ret <2 x half> %5
+}
+
+define <2 x half> @fma_f16x2_maxnum_no_nans(<2 x half> %a, <2 x half> %b, <2 x half> %c) {
+; CHECK-LABEL: fma_f16x2_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_f16x2_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.ftz.relu.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_f16x2_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .b16 %rs<5>;
+; CHECK-SM70-NEXT: .reg .b32 %r<6>;
+; CHECK-SM70-NEXT: .reg .f32 %f<5>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_f16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_f16x2_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_f16x2_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: fma.rn.f16x2 %r4, %r3, %r2, %r1;
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-SM70-NEXT: cvt.f32.f16 %f1, %rs2;
+; CHECK-SM70-NEXT: max.f32 %f2, %f1, 0f00000000;
+; CHECK-SM70-NEXT: cvt.rn.f16.f32 %rs3, %f2;
+; CHECK-SM70-NEXT: cvt.f32.f16 %f3, %rs1;
+; CHECK-SM70-NEXT: max.f32 %f4, %f3, 0f00000000;
+; CHECK-SM70-NEXT: cvt.rn.f16.f32 %rs4, %f4;
+; CHECK-SM70-NEXT: mov.b32 %r5, {%rs4, %rs3};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r5;
+; CHECK-SM70-NEXT: ret;
+ %1 = call nnan <2 x half> @llvm.fma.f16x2(<2 x half> %a, <2 x half> %b, <2 x half> %c)
+ %2 = call nsz <2 x half> @llvm.maxnum.f16x2(<2 x half> %1, <2 x half> <half 0.0, half 0.0>)
+ ret <2 x half> %2
+}
+
+define <2 x bfloat> @fma_bf16x2_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
+; CHECK-LABEL: fma_bf16x2_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<5>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<19>;
+; CHECK-SM70-NEXT: .reg .b32 %r<31>;
+; CHECK-SM70-NEXT: .reg .f32 %f<11>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs10}, %r15; }
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r27; }
+; CHECK-SM70-NEXT: and.b32 %r28, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: setp.gt.f32 %p3, %f9, 0f00000000;
+; CHECK-SM70-NEXT: and.b32 %r29, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f10, %r29;
+; CHECK-SM70-NEXT: setp.gt.f32 %p4, %f10, 0f00000000;
+; CHECK-SM70-NEXT: selp.b16 %rs17, %rs15, 0x0000, %p4;
+; CHECK-SM70-NEXT: selp.b16 %rs18, %rs10, 0x0000, %p3;
+; CHECK-SM70-NEXT: mov.b32 %r30, {%rs18, %rs17};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r30;
+; CHECK-SM70-NEXT: ret;
+ %1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
+ %2 = fcmp nsz ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
+ %3 = select nsz <2 x i1> %2, <2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>
+ ret <2 x bfloat> %3
+}
+
+; FMA_relu shouldn't be selected if the FMA operation has multiple uses
+define <2 x bfloat> @fma_bf16x2_no_nans_multiple_uses_of_fma(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
+; CHECK-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<11>;
+; CHECK-NEXT: .reg .b32 %r<14>;
+; CHECK-NEXT: .reg .f32 %f<9>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-NEXT: mov.b32 %f1, %r6;
+; CHECK-NEXT: add.rn.f32 %f2, %f1, 0f40E00000;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
+; CHECK-NEXT: cvt.u32.u16 %r7, %rs2;
+; CHECK-NEXT: shl.b32 %r8, %r7, 16;
+; CHECK-NEXT: mov.b32 %f3, %r8;
+; CHECK-NEXT: add.rn.f32 %f4, %f3, 0f40E00000;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs6, %f4;
+; CHECK-NEXT: cvt.u32.u16 %r9, %rs6;
+; CHECK-NEXT: shl.b32 %r10, %r9, 16;
+; CHECK-NEXT: mov.b32 %f5, %r10;
+; CHECK-NEXT: add.rn.f32 %f6, %f5, %f3;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs8, %f6;
+; CHECK-NEXT: cvt.u32.u16 %r11, %rs4;
+; CHECK-NEXT: shl.b32 %r12, %r11, 16;
+; CHECK-NEXT: mov.b32 %f7, %r12;
+; CHECK-NEXT: add.rn.f32 %f8, %f7, %f1;
+; CHECK-NEXT: cvt.rn.bf16.f32 %rs10, %f8;
+; CHECK-NEXT: mov.b32 %r13, {%rs10, %rs8};
+; CHECK-NEXT: st.param.b32 [func_retval0], %r13;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<11>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<14>;
+; CHECK-FTZ-NEXT: .reg .f32 %f<9>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-FTZ-NEXT: fma.rn.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r4;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r5, %rs1;
+; CHECK-FTZ-NEXT: shl.b32 %r6, %r5, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f1, %r6;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f2, %f1, 0f40E00000;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs4, %f2;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r7, %rs2;
+; CHECK-FTZ-NEXT: shl.b32 %r8, %r7, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f3, %r8;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f4, %f3, 0f40E00000;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs6, %f4;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r9, %rs6;
+; CHECK-FTZ-NEXT: shl.b32 %r10, %r9, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f5, %r10;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f6, %f5, %f3;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs8, %f6;
+; CHECK-FTZ-NEXT: cvt.u32.u16 %r11, %rs4;
+; CHECK-FTZ-NEXT: shl.b32 %r12, %r11, 16;
+; CHECK-FTZ-NEXT: mov.b32 %f7, %r12;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32 %f8, %f7, %f1;
+; CHECK-FTZ-NEXT: cvt.rn.bf16.f32 %rs10, %f8;
+; CHECK-FTZ-NEXT: mov.b32 %r13, {%rs10, %rs8};
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r13;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_no_nans_multiple_uses_of_fma(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<7>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<17>;
+; CHECK-SM70-NEXT: .reg .b32 %r<57>;
+; CHECK-SM70-NEXT: .reg .f32 %f<17>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_no_nans_multiple_uses_of_fma_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: and.b32 %r28, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: add.rn.f32 %f10, %f9, 0f40E00000;
+; CHECK-SM70-NEXT: mov.b32 %r29, %f10;
+; CHECK-SM70-NEXT: bfe.u32 %r30, %r29, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r29;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %f10, %f10;
+; CHECK-SM70-NEXT: or.b32 %r33, %r29, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
+; CHECK-SM70-NEXT: and.b32 %r35, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f11, %r35;
+; CHECK-SM70-NEXT: add.rn.f32 %f12, %f11, 0f40E00000;
+; CHECK-SM70-NEXT: mov.b32 %r36, %f12;
+; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %f12, %f12;
+; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p4;
+; CHECK-SM70-NEXT: and.b32 %r42, %r41, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f13, %r42;
+; CHECK-SM70-NEXT: add.rn.f32 %f14, %f13, %f11;
+; CHECK-SM70-NEXT: mov.b32 %r43, %f14;
+; CHECK-SM70-NEXT: bfe.u32 %r44, %r43, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r45, %r44, %r43;
+; CHECK-SM70-NEXT: add.s32 %r46, %r45, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p5, %f14, %f14;
+; CHECK-SM70-NEXT: or.b32 %r47, %r43, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r48, %r47, %r46, %p5;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs13}, %r48; }
+; CHECK-SM70-NEXT: and.b32 %r49, %r34, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f15, %r49;
+; CHECK-SM70-NEXT: add.rn.f32 %f16, %f15, %f9;
+; CHECK-SM70-NEXT: mov.b32 %r50, %f16;
+; CHECK-SM70-NEXT: bfe.u32 %r51, %r50, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r52, %r51, %r50;
+; CHECK-SM70-NEXT: add.s32 %r53, %r52, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p6, %f16, %f16;
+; CHECK-SM70-NEXT: or.b32 %r54, %r50, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r55, %r54, %r53, %p6;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r55; }
+; CHECK-SM70-NEXT: mov.b32 %r56, {%rs15, %rs13};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r56;
+; CHECK-SM70-NEXT: ret;
+ %1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
+ %2 = fcmp nsz ogt <2 x bfloat> %1, <bfloat 0.0, bfloat 0.0>
+ %3 = select nsz <2 x i1> %2, <2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>
+ %4 = fadd <2 x bfloat> %1, <bfloat 7.0, bfloat 7.0>
+ %5 = fadd <2 x bfloat> %4, %1
+ ret <2 x bfloat> %5
+}
+
+define <2 x bfloat> @fma_bf16x2_maxnum_no_nans(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c) {
+; CHECK-LABEL: fma_bf16x2_maxnum_no_nans(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-NEXT: ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
+; CHECK-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_0];
+; CHECK-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: fma_bf16x2_maxnum_no_nans(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-FTZ-NEXT: ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
+; CHECK-FTZ-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_0];
+; CHECK-FTZ-NEXT: fma.rn.relu.bf16x2 %r4, %r3, %r2, %r1;
+; CHECK-FTZ-NEXT: st.param.b32 [func_retval0], %r4;
+; CHECK-FTZ-NEXT: ret;
+;
+; CHECK-SM70-LABEL: fma_bf16x2_maxnum_no_nans(
+; CHECK-SM70: {
+; CHECK-SM70-NEXT: .reg .pred %p<5>;
+; CHECK-SM70-NEXT: .reg .b16 %rs<17>;
+; CHECK-SM70-NEXT: .reg .b32 %r<43>;
+; CHECK-SM70-NEXT: .reg .f32 %f<13>;
+; CHECK-SM70-EMPTY:
+; CHECK-SM70-NEXT: // %bb.0:
+; CHECK-SM70-NEXT: ld.param.b32 %r1, [fma_bf16x2_maxnum_no_nans_param_0];
+; CHECK-SM70-NEXT: ld.param.b32 %r2, [fma_bf16x2_maxnum_no_nans_param_1];
+; CHECK-SM70-NEXT: ld.param.b32 %r3, [fma_bf16x2_maxnum_no_nans_param_2];
+; CHECK-SM70-NEXT: mov.b32 {%rs1, %rs2}, %r3;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r4, %rs1;
+; CHECK-SM70-NEXT: shl.b32 %r5, %r4, 16;
+; CHECK-SM70-NEXT: mov.b32 %f1, %r5;
+; CHECK-SM70-NEXT: mov.b32 {%rs4, %rs5}, %r2;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r6, %rs4;
+; CHECK-SM70-NEXT: shl.b32 %r7, %r6, 16;
+; CHECK-SM70-NEXT: mov.b32 %f2, %r7;
+; CHECK-SM70-NEXT: mov.b32 {%rs7, %rs8}, %r1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r8, %rs7;
+; CHECK-SM70-NEXT: shl.b32 %r9, %r8, 16;
+; CHECK-SM70-NEXT: mov.b32 %f3, %r9;
+; CHECK-SM70-NEXT: fma.rn.f32 %f4, %f3, %f2, %f1;
+; CHECK-SM70-NEXT: mov.b32 %r10, %f4;
+; CHECK-SM70-NEXT: bfe.u32 %r11, %r10, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r12, %r11, %r10;
+; CHECK-SM70-NEXT: add.s32 %r13, %r12, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p1, %f4, %f4;
+; CHECK-SM70-NEXT: or.b32 %r14, %r10, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r15, %r14, %r13, %p1;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r16, %rs2;
+; CHECK-SM70-NEXT: shl.b32 %r17, %r16, 16;
+; CHECK-SM70-NEXT: mov.b32 %f5, %r17;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r18, %rs5;
+; CHECK-SM70-NEXT: shl.b32 %r19, %r18, 16;
+; CHECK-SM70-NEXT: mov.b32 %f6, %r19;
+; CHECK-SM70-NEXT: cvt.u32.u16 %r20, %rs8;
+; CHECK-SM70-NEXT: shl.b32 %r21, %r20, 16;
+; CHECK-SM70-NEXT: mov.b32 %f7, %r21;
+; CHECK-SM70-NEXT: fma.rn.f32 %f8, %f7, %f6, %f5;
+; CHECK-SM70-NEXT: mov.b32 %r22, %f8;
+; CHECK-SM70-NEXT: bfe.u32 %r23, %r22, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r24, %r23, %r22;
+; CHECK-SM70-NEXT: add.s32 %r25, %r24, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p2, %f8, %f8;
+; CHECK-SM70-NEXT: or.b32 %r26, %r22, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r27, %r26, %r25, %p2;
+; CHECK-SM70-NEXT: and.b32 %r28, %r27, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f9, %r28;
+; CHECK-SM70-NEXT: max.f32 %f10, %f9, 0f00000000;
+; CHECK-SM70-NEXT: mov.b32 %r29, %f10;
+; CHECK-SM70-NEXT: bfe.u32 %r30, %r29, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r31, %r30, %r29;
+; CHECK-SM70-NEXT: add.s32 %r32, %r31, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p3, %f10, %f10;
+; CHECK-SM70-NEXT: or.b32 %r33, %r29, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r34, %r33, %r32, %p3;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs13}, %r34; }
+; CHECK-SM70-NEXT: and.b32 %r35, %r15, -65536;
+; CHECK-SM70-NEXT: mov.b32 %f11, %r35;
+; CHECK-SM70-NEXT: max.f32 %f12, %f11, 0f00000000;
+; CHECK-SM70-NEXT: mov.b32 %r36, %f12;
+; CHECK-SM70-NEXT: bfe.u32 %r37, %r36, 16, 1;
+; CHECK-SM70-NEXT: add.s32 %r38, %r37, %r36;
+; CHECK-SM70-NEXT: add.s32 %r39, %r38, 32767;
+; CHECK-SM70-NEXT: setp.nan.f32 %p4, %f12, %f12;
+; CHECK-SM70-NEXT: or.b32 %r40, %r36, 4194304;
+; CHECK-SM70-NEXT: selp.b32 %r41, %r40, %r39, %p4;
+; CHECK-SM70-NEXT: { .reg .b16 tmp; mov.b32 {tmp, %rs15}, %r41; }
+; CHECK-SM70-NEXT: mov.b32 %r42, {%rs15, %rs13};
+; CHECK-SM70-NEXT: st.param.b32 [func_retval0], %r42;
+; CHECK-SM70-NEXT: ret;
+ %1 = call nnan <2 x bfloat> @llvm.fma.bf16x2(<2 x bfloat> %a, <2 x bfloat> %b, <2 x bfloat> %c)
+ %2 = call nsz <2 x bfloat> @llvm.maxnum.bf16x2(<2 x bfloat> %1, <2 x bfloat> <bfloat 0.0, bfloat 0.0>)
+ ret <2 x bfloat> %2
+}
More information about the llvm-commits
mailing list