[llvm] 6c3604b - [NVPTX] Add support for f32x2 mixed-precision add/sub (#221957)
via llvm-commits
llvm-commits at lists.llvm.org
Thu Sep 17 02:30:47 PDT 2026
Author: Srinivasa Ravi
Date: 2026-09-17T15:00:40+05:30
New Revision: 6c3604b43ca23b305c10b60ed9120ea56f872233
URL: https://github.com/llvm/llvm-project/commit/6c3604b43ca23b305c10b60ed9120ea56f872233
DIFF: https://github.com/llvm/llvm-project/commit/6c3604b43ca23b305c10b60ed9120ea56f872233.diff
LOG: [NVPTX] Add support for f32x2 mixed-precision add/sub (#221957)
This change adds support for mixed precision addition and
subtraction of `f16x2` and `bf16x2` with `f32x2`, where the
following upconverting patterns:
```
%e = fpext <2 x half> %h to <2 x float>
%res = fp-operation(%e, ...)
...
%e = fpext <2 x bfloat> %b to <2 x float>
%res = fp-operation(%e, ...)
where the fp-operation can be any of:
- fadd
- fsub
- llvm.nvvm.fadd.v2f32
```
are lowered to `add/sub.{rnd}.f32x2.{f16x2/bf16x2}.f32x2`, and the
following downconverting pattern:
```
%sum = llvm.nvvm.fadd{.ftz}.v2f32(%a, %b, rz)
%lo = extractelement <2 x float> %sum, i32 0
%hi = extractelement <2 x float> %sum, i32 1
%res = llvm.nvvm.ff2{f16x2/bf16x2}.rz(%hi, %lo, false)
```
is lowered to `add/sub.rz{.ftz}.{f16x2/bf16x2}.f32x2.f32x2`. These
instructions combine the conversion and the addition into one
instruction from `sm_107f` onwards. `fneg` followed by one of the
above addition patterns is lowered to the corresponding `sub`
instruction.
The upconverting patterns are folded only when atleast one of the
`fpext`
operands have a single use, thus purely eliminating the conversion.
The downconverting patterns are folded only when the `f32x2` result
is used solely by the conversion, avoiding generating an unnecessary
additional `add/sub` instruction.
The tests have been verified through ptxas-13.4.
PTX spec reference:
https://docs.nvidia.com/cuda/developer-preview/13.4/parallel-thread-execution/index.html#mixed-precision-floating-point-instructions-add
Added:
llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert-invalid.ll
llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert.ll
llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert-invalid.ll
llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll
llvm/test/CodeGen/NVPTX/mixed-precision-f32x2-sm-version.ll
llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert-invalid.ll
llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert.ll
llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert-invalid.ll
llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert.ll
Modified:
llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
Removed:
################################################################################
diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 9695779430d23..1ae80f4937fa8 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -265,6 +265,8 @@ def hasConvertWithPZOSupport : PredOr<[hasRubinFamilySupport]>;
def hasMBarrierLayoutSupport : PredAnd<[PTX93, SM90]>;
+def hasFP16x2MixedPrecisionSupport : PredOr<[hasRubinFamilySupport]>;
+
//===----------------------------------------------------------------------===//
// Some Common Instruction Class Templates
//===----------------------------------------------------------------------===//
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 8fd05c58087e1..f54fd1b348af8 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2614,9 +2614,38 @@ foreach ftz = ["", "ftz"] in {
def INT_NVVM_ADD_D :
F_MATH_2_RNDOP_TY<"add.${rnd}.f64", F64RT, int_nvvm_fadd>;
-// mixed precision
+// Mixed Precision
+
+// Match only when the fold removes at least one explicit conversion, and the
+// extended vector is not shared.
+class FoldableFPExtendV2F32<dag ops, dag frag>
+ : PatFrag<ops, frag, [{
+ return N->hasOneUse() && (N->getOperand(0)->hasOneUse() ||
+ N->getOperand(1)->hasOneUse());
+ }]>;
+
+// FP_EXTEND to v2f32 is scalarized before ISEL, leaving one of two patterns
+// behind depending on whether the source lanes share a register.
+def fpextend_v2f32_from_packed
+ : FoldableFPExtendV2F32<(ops node:$a),
+ (v2f32 (build_vector
+ (f32 (fpextend (extractelt node:$a, 0))),
+ (f32 (fpextend (extractelt node:$a, 1)))))>;
+
+def fpextend_v2f32_from_lanes
+ : FoldableFPExtendV2F32<(ops node:$a0, node:$a1),
+ (v2f32 (build_vector
+ (f32 (fpextend node:$a0)),
+ (f32 (fpextend node:$a1))))>;
+
+// Fuse downconverting add/sub only when the result of the add/sub is solely
+// used for downconverting.
+def extractelt_only_for_cvt
+ : PatFrag<(ops node:$vec, node:$lane), (extractelt node:$vec, node:$lane),
+ [{ return N->hasOneUse() && N->getOperand(0)->hasNUsesOfValue(2, 0); }]>;
+
foreach rnd = FPRoundingModes in {
- defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
+ defvar rnd_imm = !cast<TImmLeaf>("fp_rnd_" # rnd # "_imm");
foreach sat = ["", "sat"] in
foreach type = [f16, bf16] in
@@ -2628,6 +2657,27 @@ foreach rnd = FPRoundingModes in {
(f32 (fpextend type:$a)),
f32:$b, rnd_imm))]>,
Requires<[SM100]>;
+
+ let Predicates = [hasFP16x2MixedPrecisionSupport] in {
+ foreach t = [F16X2RT, BF16X2RT] in {
+ def INT_NVVM_MIXED_ADD_ # rnd # _f32x2_ # t.PtxType :
+ BasicNVPTXInst<(outs B64:$dst), (ins B32:$a, B64:$b),
+ "add." # rnd # ".f32x2." # t.PtxType # ".f32x2">;
+
+ defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_ADD_" # rnd #
+ "_f32x2_" # t.PtxType);
+
+ def : Pat<(v2f32 (int_nvvm_fadd (fpextend_v2f32_from_packed t.Ty:$a),
+ v2f32:$b, rnd_imm)),
+ (Inst $a, $b)>;
+
+ def : Pat<(v2f32 (int_nvvm_fadd
+ (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
+ t.Ty.ElementType:$a1),
+ v2f32:$b, rnd_imm)),
+ (Inst (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
+ }
+ }
}
// Pattern for fadd when there is no FTZ flag
@@ -2638,6 +2688,45 @@ let Predicates = [SM100, doNoF32FTZ] in {
(INT_NVVM_MIXED_ADD_rn_f32_bf16 B16:$a, B32:$b)>;
}
+let Predicates = [hasFP16x2MixedPrecisionSupport, doNoF32FTZ] in
+ foreach t = [F16X2RT, BF16X2RT] in {
+ defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_ADD_rn_f32x2_" # t.PtxType);
+
+ def : Pat<(v2f32 (fadd (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b)),
+ (Inst $a, $b)>;
+
+ def : Pat<(v2f32 (fadd (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
+ t.Ty.ElementType:$a1),
+ v2f32:$b)),
+ (Inst (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
+ }
+
+// Mixed precision - Downconverting
+
+// f16x2 - rz rounding mode, only with ftz
+// bf16x2 - rz rounding mode, without ftz
+let Predicates = [hasFP16x2MixedPrecisionSupport] in
+ foreach t = [F16X2RT, BF16X2RT] in {
+ defvar ftz = !if(!eq(t, F16X2RT), ".ftz", "");
+ defvar AddOp = !cast<Intrinsic>("int_nvvm_fadd" # !if(!eq(ftz, ""), "", "_ftz"));
+ defvar CvtOp = !cast<Intrinsic>("int_nvvm_ff2" # t.PtxType # "_rz");
+
+ def INT_NVVM_MIXED_ADD_rz_ # t.PtxType # _f32x2 :
+ BasicNVPTXInst<(outs B32:$dst), (ins B64:$a, B64:$b),
+ "add.rz" # ftz # "." # t.PtxType # ".f32x2.f32x2">;
+
+ defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_ADD_rz_" # t.PtxType #
+ "_f32x2");
+
+ // The cvt packs its first argument into the upper half of the result.
+ def : Pat<(CvtOp (extractelt_only_for_cvt
+ (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 1),
+ (extractelt_only_for_cvt
+ (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 0),
+ /*pzo=*/0),
+ (Inst $a, $b)>;
+ }
+
//
// Sub
//
@@ -2698,9 +2787,10 @@ def INT_NVVM_SUB_D :
(ins FPRoundingMode:$rnd), "sub.${rnd}.f64",
[(set f64:$dst, (int_nvvm_fadd f64:$a, (f64 (fneg f64:$b)), timm:$rnd))]>;
-// mixed precision
+// Mixed Precision
+
foreach rnd = FPRoundingModes in {
- defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
+ defvar rnd_imm = !cast<TImmLeaf>("fp_rnd_" # rnd # "_imm");
foreach sat = ["", "sat"] in
foreach type = [f16, bf16] in
@@ -2712,6 +2802,29 @@ foreach rnd = FPRoundingModes in {
(f32 (fpextend type:$a)),
(f32 (fneg f32:$b)), rnd_imm))]>,
Requires<[SM100]>;
+
+ let Predicates = [hasFP16x2MixedPrecisionSupport] in {
+ foreach t = [F16X2RT, BF16X2RT] in {
+ // combineFAddWithNeg has already folded the fneg into a sub node.
+ defvar SubOp = !cast<SDNode>("sub_" # rnd);
+
+ def INT_NVVM_MIXED_SUB_ # rnd # _f32x2_ # t.PtxType :
+ BasicNVPTXInst<(outs B64:$dst), (ins B32:$a, B64:$b),
+ "sub." # rnd # ".f32x2." # t.PtxType # ".f32x2">;
+
+ defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_SUB_" # rnd #
+ "_f32x2_" # t.PtxType);
+
+ def : Pat<(v2f32 (SubOp (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b)),
+ (Inst $a, $b)>;
+
+ def : Pat<(v2f32 (SubOp
+ (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
+ t.Ty.ElementType:$a1),
+ v2f32:$b)),
+ (Inst (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
+ }
+ }
}
// Pattern for fsub when there is no FTZ flag
@@ -2722,6 +2835,43 @@ let Predicates = [SM100, doNoF32FTZ] in {
(INT_NVVM_MIXED_SUB_rn_f32_bf16 B16:$a, B32:$b)>;
}
+let Predicates = [hasFP16x2MixedPrecisionSupport, doNoF32FTZ] in
+ foreach t = [F16X2RT, BF16X2RT] in {
+ defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_SUB_rn_f32x2_" # t.PtxType);
+
+ def : Pat<(v2f32 (fsub (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b)),
+ (Inst $a, $b)>;
+
+ def : Pat<(v2f32 (fsub (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
+ t.Ty.ElementType:$a1),
+ v2f32:$b)),
+ (Inst (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
+ }
+
+// Mixed precision - Downconverting
+
+// f16x2 - rz rounding mode, only with ftz
+// bf16x2 - rz rounding mode, without ftz
+let Predicates = [hasFP16x2MixedPrecisionSupport] in
+ foreach t = [F16X2RT, BF16X2RT] in {
+ defvar ftz = !if(!eq(t, F16X2RT), ".ftz", "");
+ defvar SubOp = !cast<SDNode>("sub_rz" # !if(!eq(ftz, ""), "", "_ftz"));
+ defvar CvtOp = !cast<Intrinsic>("int_nvvm_ff2" # t.PtxType # "_rz");
+
+ def INT_NVVM_MIXED_SUB_rz_ # t.PtxType # _f32x2 :
+ BasicNVPTXInst<(outs B32:$dst), (ins B64:$a, B64:$b),
+ "sub.rz" # ftz # "." # t.PtxType # ".f32x2.f32x2">;
+
+ defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_SUB_rz_" # t.PtxType #
+ "_f32x2");
+
+ // The cvt packs its first argument into the upper half of the result.
+ def : Pat<(CvtOp (extractelt_only_for_cvt (SubOp v2f32:$a, v2f32:$b), 1),
+ (extractelt_only_for_cvt (SubOp v2f32:$a, v2f32:$b), 0),
+ /*pzo=*/0),
+ (Inst $a, $b)>;
+ }
+
//
// BFIND
//
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert-invalid.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert-invalid.ll
new file mode 100644
index 0000000000000..03ad29a64d20e
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert-invalid.ll
@@ -0,0 +1,262 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+;
+; Cases where the mixed-precision instruction must not be selected.
+;
+
+;
+; F16x2
+;
+
+define <2 x half> @add_rz_f16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_extra_use(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_extra_use_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_extra_use_param_1];
+; CHECK-NEXT: add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: ld.param::func.b64 %rd4, [add_rz_f16x2_f32x2_extra_use_param_2];
+; CHECK-NEXT: st.b64 [%rd4], %rd3;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ ; extra use
+ store <2 x float> %sum, ptr %p
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+; unsupported forms
+
+define <2 x half> @add_rz_f16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_swapped_halves(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_swapped_halves_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_swapped_halves_param_1];
+; CHECK-NEXT: add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.f16x2.f32 %r3, %r1, %r2;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %lo, float %hi, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_no_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_no_ftz(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_no_ftz_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_no_ftz_param_1];
+; CHECK-NEXT: add.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x half> @add_rn_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rn_f16x2_f32x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rn_f16x2_f32x2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rn_f16x2_f32x2_param_1];
+; CHECK-NEXT: add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_rn_convert(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_rn_convert(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_rn_convert_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_rn_convert_param_1];
+; CHECK-NEXT: add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rn.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_relu(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_relu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_relu_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_relu_param_1];
+; CHECK-NEXT: add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.relu.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_distinct_adds(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_distinct_adds(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_distinct_adds_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_distinct_adds_param_1];
+; CHECK-NEXT: add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: ld.param::func.b64 %rd4, [add_rz_f16x2_f32x2_distinct_adds_param_2];
+; CHECK-NEXT: add.rz.ftz.f32x2 %rd5, %rd1, %rd4;
+; CHECK-NEXT: mov.b64 {%r1, _}, %rd3;
+; CHECK-NEXT: mov.b64 {_, %r2}, %rd5;
+; CHECK-NEXT: cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %sum1 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ %sum2 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %c, i32 0)
+ %lo = extractelement <2 x float> %sum1, i32 0
+ %hi = extractelement <2 x float> %sum2, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+;
+; BF16x2
+;
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2_extra_use(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_extra_use_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_extra_use_param_1];
+; CHECK-NEXT: add.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: ld.param::func.b64 %rd4, [add_rz_bf16x2_f32x2_extra_use_param_2];
+; CHECK-NEXT: st.b64 [%rd4], %rd3;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ ; extra use
+ store <2 x float> %sum, ptr %p
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x bfloat> %r
+}
+
+; unsupported forms
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2_swapped_halves(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_swapped_halves_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_swapped_halves_param_1];
+; CHECK-NEXT: add.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.bf16x2.f32 %r3, %r1, %r2;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %lo, float %hi, i1 false)
+ ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2_ftz(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_ftz_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_ftz_param_1];
+; CHECK-NEXT: add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @add_rn_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rn_bf16x2_f32x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rn_bf16x2_f32x2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rn_bf16x2_f32x2_param_1];
+; CHECK-NEXT: add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x bfloat> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert.ll
new file mode 100644
index 0000000000000..de26b06fcbdf2
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert.ll
@@ -0,0 +1,87 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+;
+; F16x2
+;
+
+define <2 x half> @add_rz_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_param_1];
+; CHECK-NEXT: add.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_commuted(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_commuted(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_commuted_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_commuted_param_1];
+; CHECK-NEXT: add.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd2, %rd1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %b, <2 x float> %a, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+;
+; BF16x2
+;
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_param_1];
+; CHECK-NEXT: add.rz.bf16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2_commuted(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2_commuted(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_commuted_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_commuted_param_1];
+; CHECK-NEXT: add.rz.bf16x2.f32x2.f32x2 %r1, %rd2, %rd1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %b, <2 x float> %a, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x bfloat> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert-invalid.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert-invalid.ll
new file mode 100644
index 0000000000000..708e9853734a7
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert-invalid.ll
@@ -0,0 +1,719 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck --check-prefixes=CHECK-NOFTZ %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | FileCheck --check-prefixes=CHECK-FTZ %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | %ptxas-verify -arch=sm_107f %}
+
+;
+; Cases where the mixed-precision instruction must not be selected.
+;
+
+;
+; F16x2
+;
+
+define <2 x float> @add_f32x2_f16x2_extra_use(<2 x half> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_extra_use(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_extra_use_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_f16x2_extra_use_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd3, [add_f32x2_f16x2_extra_use_param_2];
+; CHECK-NOFTZ-NEXT: st.b64 [%rd3], %rd1;
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_extra_use(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_extra_use_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_f16x2_extra_use_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd3, [add_f32x2_f16x2_extra_use_param_2];
+; CHECK-FTZ-NEXT: st.b64 [%rd3], %rd1;
+; CHECK-FTZ-NEXT: add.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ ; extra use
+ store <2 x float> %e, ptr %p
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_f16x2_generic_lanes_multiuse(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_lanes_multiuse(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lanes_multiuse_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_lanes_multiuse_param_1];
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd4, %rd1, %rd3;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_lanes_multiuse(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lanes_multiuse_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_lanes_multiuse_param_1];
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd4, %rd1, %rd3;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ %r1 = fadd <2 x float> %e, %b
+ ; extra use
+ %r2 = fadd <2 x float> %e, %r1
+ ret <2 x float> %r2
+}
+
+define <2 x float> @add_f32x2_f16x2_generic_packed_multiuse(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_packed_multiuse(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_f16x2_generic_packed_multiuse_param_0];
+; CHECK-NOFTZ-NEXT: mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_packed_multiuse_param_1];
+; CHECK-NOFTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r3, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r4, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd2, {%r4, %r3};
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd4, %rd2, %rd3;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_packed_multiuse(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_f16x2_generic_packed_multiuse_param_0];
+; CHECK-FTZ-NEXT: mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_packed_multiuse_param_1];
+; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r3, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r4, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd4, %rd2, %rd3;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %x = fmul <2 x half> %a, %a
+ %e = fpext <2 x half> %x to <2 x float>
+ %r1 = fadd <2 x float> %e, %b
+ ; extra use
+ %r2 = fadd <2 x float> %e, %r1
+ ret <2 x float> %r2
+}
+
+define <2 x float> @add_f32x2_f16x2_generic_both_lanes_extra_use(<2 x half> %a, <2 x float> %b, ptr %p, ptr %q) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_both_lanes_extra_use(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd3, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r2, %rs2;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd4, {%r1, %r2};
+; CHECK-NOFTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT: st.b32 [%rd3], %r2;
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd5, %rd4, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_both_lanes_extra_use(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd3, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs2;
+; CHECK-FTZ-NEXT: mov.b64 %rd4, {%r1, %r2};
+; CHECK-FTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT: st.b32 [%rd3], %r2;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd5, %rd4, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT: ret;
+ %a0 = extractelement <2 x half> %a, i32 0
+ %a1 = extractelement <2 x half> %a, i32 1
+ %e0 = fpext half %a0 to float
+ %e1 = fpext half %a1 to float
+ %v0 = insertelement <2 x float> poison, float %e0, i32 0
+ %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+ ; extra uses
+ store float %e0, ptr %p
+ store float %e1, ptr %q
+ %r = fadd <2 x float> %v1, %b
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_sub_f32x2_f16x2_generic_multiuse(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_sub_f32x2_f16x2_generic_multiuse(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_f16x2_generic_multiuse_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_sub_f32x2_f16x2_generic_multiuse_param_1];
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd5, %rd3, %rd4;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_sub_f32x2_f16x2_generic_multiuse(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_f16x2_generic_multiuse_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_sub_f32x2_f16x2_generic_multiuse_param_1];
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd5, %rd3, %rd4;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ %add = fadd <2 x float> %e, %b
+ ; extra use, as in llvm/llvm-project#213327
+ %sub = fsub <2 x float> %e, %b
+ %r = fadd <2 x float> %add, %sub
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_sub_f32x2_f16x2_multiuse(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_sub_f32x2_f16x2_multiuse(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_f16x2_multiuse_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_sub_f32x2_f16x2_multiuse_param_1];
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd5, %rd3, %rd4;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_sub_f32x2_f16x2_multiuse(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_f16x2_multiuse_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_sub_f32x2_f16x2_multiuse_param_1];
+; CHECK-FTZ-NEXT: add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT: add.rn.f32x2 %rd5, %rd3, %rd4;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ %add = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+ %nb = fneg <2 x float> %b
+ ; extra use, as in llvm/llvm-project#213327
+ %sub = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %add, <2 x float> %sub, i32 1)
+ ret <2 x float> %r
+}
+
+; unsupported forms
+
+define <2 x float> @add_f32x2_f16x2_ftz(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_ftz(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_ftz_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_f16x2_ftz_param_1];
+; CHECK-NOFTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_ftz(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_ftz_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_f16x2_ftz_param_1];
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_f16x2_partial(<2 x half> %a, float %c, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_partial(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<2>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_f16x2_partial_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b16 %rs1, [add_f32x2_f16x2_partial_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_partial_param_2];
+; CHECK-NOFTZ-NEXT: mov.b64 %rd2, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_partial(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<2>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_f16x2_partial_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b16 %rs1, [add_f32x2_f16x2_partial_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_partial_param_2];
+; CHECK-FTZ-NEXT: mov.b64 %rd2, {%r2, %r1};
+; CHECK-FTZ-NEXT: add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %a0 = extractelement <2 x half> %a, i32 0
+ %e0 = fpext half %a0 to float
+ %v0 = insertelement <2 x float> poison, float %e0, i32 0
+ %v1 = insertelement <2 x float> %v0, float %c, i32 1
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %v1, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+;
+; BF16x2
+;
+
+define <2 x float> @add_f32x2_bf16x2_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_extra_use(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_extra_use_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_bf16x2_extra_use_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd3, [add_f32x2_bf16x2_extra_use_param_2];
+; CHECK-NOFTZ-NEXT: st.b64 [%rd3], %rd1;
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_extra_use(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_extra_use_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_bf16x2_extra_use_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd3, [add_f32x2_bf16x2_extra_use_param_2];
+; CHECK-FTZ-NEXT: st.b64 [%rd3], %rd1;
+; CHECK-FTZ-NEXT: add.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ ; extra use
+ store <2 x float> %e, ptr %p
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2_generic_lanes_multiuse(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_lanes_multiuse(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lanes_multiuse_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_lanes_multiuse_param_1];
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd4, %rd1, %rd3;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_lanes_multiuse(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lanes_multiuse_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_lanes_multiuse_param_1];
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd4, %rd1, %rd3;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ %r1 = fadd <2 x float> %e, %b
+ ; extra use
+ %r2 = fadd <2 x float> %e, %r1
+ ret <2 x float> %r2
+}
+
+define <2 x float> @add_f32x2_bf16x2_generic_packed_multiuse(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_packed_multiuse(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_bf16x2_generic_packed_multiuse_param_0];
+; CHECK-NOFTZ-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_packed_multiuse_param_1];
+; CHECK-NOFTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r3, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r4, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd2, {%r4, %r3};
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd4, %rd2, %rd3;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_packed_multiuse(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_bf16x2_generic_packed_multiuse_param_0];
+; CHECK-FTZ-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_packed_multiuse_param_1];
+; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r3, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r4, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd4, %rd2, %rd3;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %x = fmul <2 x bfloat> %a, %a
+ %e = fpext <2 x bfloat> %x to <2 x float>
+ %r1 = fadd <2 x float> %e, %b
+ ; extra use
+ %r2 = fadd <2 x float> %e, %r1
+ ret <2 x float> %r2
+}
+
+define <2 x float> @add_f32x2_bf16x2_generic_both_lanes_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p, ptr %q) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_both_lanes_extra_use(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd3, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r2, %rs2;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd4, {%r1, %r2};
+; CHECK-NOFTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT: st.b32 [%rd3], %r2;
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd5, %rd4, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_both_lanes_extra_use(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd3, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs2;
+; CHECK-FTZ-NEXT: mov.b64 %rd4, {%r1, %r2};
+; CHECK-FTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT: st.b32 [%rd3], %r2;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd5, %rd4, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT: ret;
+ %a0 = extractelement <2 x bfloat> %a, i32 0
+ %a1 = extractelement <2 x bfloat> %a, i32 1
+ %e0 = fpext bfloat %a0 to float
+ %e1 = fpext bfloat %a1 to float
+ %v0 = insertelement <2 x float> poison, float %e0, i32 0
+ %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+ ; extra uses
+ store float %e0, ptr %p
+ store float %e1, ptr %q
+ %r = fadd <2 x float> %v1, %b
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_sub_f32x2_bf16x2_generic_multiuse(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_sub_f32x2_bf16x2_generic_multiuse(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_bf16x2_generic_multiuse_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_sub_f32x2_bf16x2_generic_multiuse_param_1];
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd5, %rd3, %rd4;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_sub_f32x2_bf16x2_generic_multiuse(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_bf16x2_generic_multiuse_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_sub_f32x2_bf16x2_generic_multiuse_param_1];
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd5, %rd3, %rd4;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ %add = fadd <2 x float> %e, %b
+ ; extra use, as in llvm/llvm-project#213327
+ %sub = fsub <2 x float> %e, %b
+ %r = fadd <2 x float> %add, %sub
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_sub_f32x2_bf16x2_multiuse(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_sub_f32x2_bf16x2_multiuse(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_bf16x2_multiuse_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_sub_f32x2_bf16x2_multiuse_param_1];
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd5, %rd3, %rd4;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_sub_f32x2_bf16x2_multiuse(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_bf16x2_multiuse_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_sub_f32x2_bf16x2_multiuse_param_1];
+; CHECK-FTZ-NEXT: add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT: add.rn.f32x2 %rd5, %rd3, %rd4;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ %add = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+ %nb = fneg <2 x float> %b
+ ; extra use, as in llvm/llvm-project#213327
+ %sub = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %add, <2 x float> %sub, i32 1)
+ ret <2 x float> %r
+}
+
+; unsupported forms
+
+define <2 x float> @add_f32x2_bf16x2_ftz(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_ftz(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_ftz_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_bf16x2_ftz_param_1];
+; CHECK-NOFTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_ftz(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_ftz_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_bf16x2_ftz_param_1];
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2_partial(<2 x bfloat> %a, float %c, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_partial(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<2>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_bf16x2_partial_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b16 %rs1, [add_f32x2_bf16x2_partial_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_partial_param_2];
+; CHECK-NOFTZ-NEXT: mov.b64 %rd2, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_partial(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<2>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_bf16x2_partial_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b16 %rs1, [add_f32x2_bf16x2_partial_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_partial_param_2];
+; CHECK-FTZ-NEXT: mov.b64 %rd2, {%r2, %r1};
+; CHECK-FTZ-NEXT: add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %a0 = extractelement <2 x bfloat> %a, i32 0
+ %e0 = fpext bfloat %a0 to float
+ %v0 = insertelement <2 x float> poison, float %e0, i32 0
+ %v1 = insertelement <2 x float> %v0, float %c, i32 1
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %v1, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll
new file mode 100644
index 0000000000000..c1acfe150289f
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll
@@ -0,0 +1,543 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck --check-prefixes=CHECK,CHECK-NOFTZ %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | FileCheck --check-prefixes=CHECK,CHECK-FTZ %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | %ptxas-verify -arch=sm_107f %}
+
+;
+; F16x2
+;
+
+; lanes
+
+define <2 x float> @add_f32x2_f16x2_lanes(<2 x half> %a0, <2 x half> %a1, <2 x half> %a2, <2 x half> %a3, <2 x float> %b) {
+; CHECK-LABEL: add_f32x2_f16x2_lanes(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<9>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_lanes_param_0];
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs3, %rs4}, [add_f32x2_f16x2_lanes_param_1];
+; CHECK-NEXT: mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_lanes_param_4];
+; CHECK-NEXT: add.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs5, %rs6}, [add_f32x2_f16x2_lanes_param_2];
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs7, %rs8}, [add_f32x2_f16x2_lanes_param_3];
+; CHECK-NEXT: mov.b32 %r2, {%rs3, %rs4};
+; CHECK-NEXT: add.rz.f32x2.f16x2.f32x2 %rd3, %r2, %rd2;
+; CHECK-NEXT: mov.b32 %r3, {%rs5, %rs6};
+; CHECK-NEXT: add.rm.f32x2.f16x2.f32x2 %rd4, %r3, %rd3;
+; CHECK-NEXT: mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT: add.rp.f32x2.f16x2.f32x2 %rd5, %r4, %rd4;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NEXT: ret;
+ %e0 = fpext <2 x half> %a0 to <2 x float>
+ %r0 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e0, <2 x float> %b, i32 1)
+ %e1 = fpext <2 x half> %a1 to <2 x float>
+ %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e1, <2 x float> %r0, i32 0)
+ %e2 = fpext <2 x half> %a2 to <2 x float>
+ %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e2, <2 x float> %r1, i32 3)
+ %e3 = fpext <2 x half> %a3 to <2 x float>
+ %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e3, <2 x float> %r2, i32 2)
+ ret <2 x float> %r3
+}
+
+define <2 x float> @add_f32x2_f16x2_lanes_commuted(<2 x half> %a, <2 x float> %b) {
+; CHECK-LABEL: add_f32x2_f16x2_lanes_commuted(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_lanes_commuted_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_lanes_commuted_param_1];
+; CHECK-NEXT: mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT: add.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %b, <2 x float> %e, i32 1)
+ ret <2 x float> %r
+}
+
+; packed
+
+define <2 x float> @add_f32x2_f16x2_packed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_packed(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_f16x2_packed_param_0];
+; CHECK-NOFTZ-NEXT: mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_packed_param_1];
+; CHECK-NOFTZ-NEXT: add.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_packed(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_f16x2_packed_param_0];
+; CHECK-FTZ-NEXT: mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_packed_param_1];
+; CHECK-FTZ-NEXT: add.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-FTZ-NEXT: ret;
+ %x = fmul <2 x half> %a, %a
+ %e = fpext <2 x half> %x to <2 x float>
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_f16x2_packed_commuted(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_packed_commuted(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_f16x2_packed_commuted_param_0];
+; CHECK-NOFTZ-NEXT: mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_packed_commuted_param_1];
+; CHECK-NOFTZ-NEXT: add.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_packed_commuted(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_f16x2_packed_commuted_param_0];
+; CHECK-FTZ-NEXT: mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_packed_commuted_param_1];
+; CHECK-FTZ-NEXT: add.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-FTZ-NEXT: ret;
+ %x = fmul <2 x half> %a, %a
+ %e = fpext <2 x half> %x to <2 x float>
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %b, <2 x float> %e, i32 1)
+ ret <2 x float> %r
+}
+
+; generic fadd
+
+define <2 x float> @add_f32x2_f16x2_generic_lanes(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_lanes(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<2>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lanes_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_lanes_param_1];
+; CHECK-NOFTZ-NEXT: mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT: add.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_lanes(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lanes_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_lanes_param_1];
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ %r = fadd <2 x float> %e, %b
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_f16x2_generic_lanes_commuted(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_lanes_commuted(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<2>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lanes_commuted_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_lanes_commuted_param_1];
+; CHECK-NOFTZ-NEXT: mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT: add.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_lanes_commuted(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lanes_commuted_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_lanes_commuted_param_1];
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ %r = fadd <2 x float> %b, %e
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_f16x2_generic_packed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_packed(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_f16x2_generic_packed_param_0];
+; CHECK-NOFTZ-NEXT: mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_packed_param_1];
+; CHECK-NOFTZ-NEXT: add.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_packed(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_f16x2_generic_packed_param_0];
+; CHECK-FTZ-NEXT: mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_packed_param_1];
+; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r3, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r4, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %x = fmul <2 x half> %a, %a
+ %e = fpext <2 x half> %x to <2 x float>
+ %r = fadd <2 x float> %e, %b
+ ret <2 x float> %r
+}
+
+; generic fadd - partially shared extension
+
+define <2 x float> @add_f32x2_f16x2_generic_lane_extra_use(<2 x half> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_lane_extra_use(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lane_extra_use_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_lane_extra_use_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_lane_extra_use_param_2];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT: mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT: add.rn.f32x2.f16x2.f32x2 %rd3, %r2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_lane_extra_use(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lane_extra_use_param_0];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_lane_extra_use_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_lane_extra_use_param_2];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs1;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs2;
+; CHECK-FTZ-NEXT: mov.b64 %rd3, {%r1, %r2};
+; CHECK-FTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd4, %rd3, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %a0 = extractelement <2 x half> %a, i32 0
+ %a1 = extractelement <2 x half> %a, i32 1
+ %e0 = fpext half %a0 to float
+ %e1 = fpext half %a1 to float
+ %v0 = insertelement <2 x float> poison, float %e0, i32 0
+ %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+ store float %e0, ptr %p
+ %r = fadd <2 x float> %v1, %b
+ ret <2 x float> %r
+}
+
+;
+; BF16x2
+;
+
+; lanes
+
+define <2 x float> @add_f32x2_bf16x2_lanes(<2 x bfloat> %a0, <2 x bfloat> %a1, <2 x bfloat> %a2, <2 x bfloat> %a3, <2 x float> %b) {
+; CHECK-LABEL: add_f32x2_bf16x2_lanes(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<9>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_lanes_param_0];
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs3, %rs4}, [add_f32x2_bf16x2_lanes_param_1];
+; CHECK-NEXT: mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_lanes_param_4];
+; CHECK-NEXT: add.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs5, %rs6}, [add_f32x2_bf16x2_lanes_param_2];
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs7, %rs8}, [add_f32x2_bf16x2_lanes_param_3];
+; CHECK-NEXT: mov.b32 %r2, {%rs3, %rs4};
+; CHECK-NEXT: add.rz.f32x2.bf16x2.f32x2 %rd3, %r2, %rd2;
+; CHECK-NEXT: mov.b32 %r3, {%rs5, %rs6};
+; CHECK-NEXT: add.rm.f32x2.bf16x2.f32x2 %rd4, %r3, %rd3;
+; CHECK-NEXT: mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT: add.rp.f32x2.bf16x2.f32x2 %rd5, %r4, %rd4;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NEXT: ret;
+ %e0 = fpext <2 x bfloat> %a0 to <2 x float>
+ %r0 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e0, <2 x float> %b, i32 1)
+ %e1 = fpext <2 x bfloat> %a1 to <2 x float>
+ %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e1, <2 x float> %r0, i32 0)
+ %e2 = fpext <2 x bfloat> %a2 to <2 x float>
+ %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e2, <2 x float> %r1, i32 3)
+ %e3 = fpext <2 x bfloat> %a3 to <2 x float>
+ %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e3, <2 x float> %r2, i32 2)
+ ret <2 x float> %r3
+}
+
+define <2 x float> @add_f32x2_bf16x2_lanes_commuted(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-LABEL: add_f32x2_bf16x2_lanes_commuted(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<3>;
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_lanes_commuted_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_lanes_commuted_param_1];
+; CHECK-NEXT: mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT: add.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %b, <2 x float> %e, i32 1)
+ ret <2 x float> %r
+}
+
+; packed
+
+define <2 x float> @add_f32x2_bf16x2_packed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-LABEL: add_f32x2_bf16x2_packed(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b32 %r1, [add_f32x2_bf16x2_packed_param_0];
+; CHECK-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_packed_param_1];
+; CHECK-NEXT: add.rn.f32x2.bf16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %x = fmul <2 x bfloat> %a, %a
+ %e = fpext <2 x bfloat> %x to <2 x float>
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2_packed_commuted(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-LABEL: add_f32x2_bf16x2_packed_commuted(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b32 %r1, [add_f32x2_bf16x2_packed_commuted_param_0];
+; CHECK-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_packed_commuted_param_1];
+; CHECK-NEXT: add.rn.f32x2.bf16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %x = fmul <2 x bfloat> %a, %a
+ %e = fpext <2 x bfloat> %x to <2 x float>
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %b, <2 x float> %e, i32 1)
+ ret <2 x float> %r
+}
+
+; generic fadd
+
+define <2 x float> @add_f32x2_bf16x2_generic_lanes(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_lanes(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<2>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lanes_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_lanes_param_1];
+; CHECK-NOFTZ-NEXT: mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT: add.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_lanes(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lanes_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_lanes_param_1];
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ %r = fadd <2 x float> %e, %b
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2_generic_lanes_commuted(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_lanes_commuted(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<2>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lanes_commuted_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_lanes_commuted_param_1];
+; CHECK-NOFTZ-NEXT: mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT: add.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_lanes_commuted(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lanes_commuted_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_lanes_commuted_param_1];
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ %r = fadd <2 x float> %b, %e
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2_generic_packed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_packed(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_bf16x2_generic_packed_param_0];
+; CHECK-NOFTZ-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_packed_param_1];
+; CHECK-NOFTZ-NEXT: add.rn.f32x2.bf16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_packed(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [add_f32x2_bf16x2_generic_packed_param_0];
+; CHECK-FTZ-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_packed_param_1];
+; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r3, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r4, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %x = fmul <2 x bfloat> %a, %a
+ %e = fpext <2 x bfloat> %x to <2 x float>
+ %r = fadd <2 x float> %e, %b
+ ret <2 x float> %r
+}
+
+; generic fadd - partially shared extension
+
+define <2 x float> @add_f32x2_bf16x2_generic_lane_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_lane_extra_use(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lane_extra_use_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_lane_extra_use_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_lane_extra_use_param_2];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT: mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT: add.rn.f32x2.bf16x2.f32x2 %rd3, %r2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_lane_extra_use(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lane_extra_use_param_0];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_lane_extra_use_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_lane_extra_use_param_2];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs1;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs2;
+; CHECK-FTZ-NEXT: mov.b64 %rd3, {%r1, %r2};
+; CHECK-FTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT: add.rn.ftz.f32x2 %rd4, %rd3, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %a0 = extractelement <2 x bfloat> %a, i32 0
+ %a1 = extractelement <2 x bfloat> %a, i32 1
+ %e0 = fpext bfloat %a0 to float
+ %e1 = fpext bfloat %a1 to float
+ %v0 = insertelement <2 x float> poison, float %e0, i32 0
+ %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+ store float %e0, ptr %p
+ %r = fadd <2 x float> %v1, %b
+ ret <2 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-f32x2-sm-version.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-f32x2-sm-version.ll
new file mode 100644
index 0000000000000..c070bbd02aad7
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-f32x2-sm-version.ll
@@ -0,0 +1,304 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck %s --check-prefixes=SM107
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_100a -mattr=+ptx94 | FileCheck %s --check-prefixes=SM100
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+; RUN: %if ptxas-sm_100a && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_100a -mattr=+ptx94 | %ptxas-verify -arch=sm_100a %}
+
+;
+; The mixed-precision f32x2 add and sub need the Rubin family. sm_100a has
+; packed f32x2 arithmetic but not the mixed forms, so it must fall back to a
+; separate cvt. One shape per instruction is enough: the fold is gated purely
+; on hasRubinFamilySupport, and the shapes themselves are covered elsewhere.
+;
+
+;
+; Upconverting
+;
+
+define <2 x float> @add_f32x2_f16x2(<2 x half> %a, <2 x float> %b) {
+; SM107-LABEL: add_f32x2_f16x2(
+; SM107: {
+; SM107-NEXT: .reg .b32 %r<3>;
+; SM107-NEXT: .reg .b64 %rd<3>;
+; SM107-EMPTY:
+; SM107-NEXT: // %bb.0:
+; SM107-NEXT: ld.param::func.b32 %r1, [add_f32x2_f16x2_param_0];
+; SM107-NEXT: mul.rn.f16x2 %r2, %r1, %r1;
+; SM107-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_param_1];
+; SM107-NEXT: add.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; SM107-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM107-NEXT: ret;
+;
+; SM100-LABEL: add_f32x2_f16x2(
+; SM100: {
+; SM100-NEXT: .reg .b16 %rs<3>;
+; SM100-NEXT: .reg .b32 %r<5>;
+; SM100-NEXT: .reg .b64 %rd<4>;
+; SM100-EMPTY:
+; SM100-NEXT: // %bb.0:
+; SM100-NEXT: ld.param::func.b32 %r1, [add_f32x2_f16x2_param_0];
+; SM100-NEXT: mul.rn.f16x2 %r2, %r1, %r1;
+; SM100-NEXT: ld.param::func.b64 %rd1, [add_f32x2_f16x2_param_1];
+; SM100-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM100-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM100-NEXT: cvt.f32.f16 %r4, %rs1;
+; SM100-NEXT: mov.b64 %rd2, {%r4, %r3};
+; SM100-NEXT: add.rn.f32x2 %rd3, %rd2, %rd1;
+; SM100-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM100-NEXT: ret;
+ %x = fmul <2 x half> %a, %a
+ %e = fpext <2 x half> %x to <2 x float>
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2(<2 x bfloat> %a, <2 x float> %b) {
+; SM107-LABEL: add_f32x2_bf16x2(
+; SM107: {
+; SM107-NEXT: .reg .b32 %r<3>;
+; SM107-NEXT: .reg .b64 %rd<3>;
+; SM107-EMPTY:
+; SM107-NEXT: // %bb.0:
+; SM107-NEXT: ld.param::func.b32 %r1, [add_f32x2_bf16x2_param_0];
+; SM107-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; SM107-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_param_1];
+; SM107-NEXT: add.rn.f32x2.bf16x2.f32x2 %rd2, %r2, %rd1;
+; SM107-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM107-NEXT: ret;
+;
+; SM100-LABEL: add_f32x2_bf16x2(
+; SM100: {
+; SM100-NEXT: .reg .b16 %rs<3>;
+; SM100-NEXT: .reg .b32 %r<5>;
+; SM100-NEXT: .reg .b64 %rd<4>;
+; SM100-EMPTY:
+; SM100-NEXT: // %bb.0:
+; SM100-NEXT: ld.param::func.b32 %r1, [add_f32x2_bf16x2_param_0];
+; SM100-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; SM100-NEXT: ld.param::func.b64 %rd1, [add_f32x2_bf16x2_param_1];
+; SM100-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM100-NEXT: cvt.f32.bf16 %r3, %rs2;
+; SM100-NEXT: cvt.f32.bf16 %r4, %rs1;
+; SM100-NEXT: mov.b64 %rd2, {%r4, %r3};
+; SM100-NEXT: add.rn.f32x2 %rd3, %rd2, %rd1;
+; SM100-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM100-NEXT: ret;
+ %x = fmul <2 x bfloat> %a, %a
+ %e = fpext <2 x bfloat> %x to <2 x float>
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_f16x2(<2 x half> %a, <2 x float> %b) {
+; SM107-LABEL: sub_f32x2_f16x2(
+; SM107: {
+; SM107-NEXT: .reg .b32 %r<3>;
+; SM107-NEXT: .reg .b64 %rd<3>;
+; SM107-EMPTY:
+; SM107-NEXT: // %bb.0:
+; SM107-NEXT: ld.param::func.b32 %r1, [sub_f32x2_f16x2_param_0];
+; SM107-NEXT: mul.rn.f16x2 %r2, %r1, %r1;
+; SM107-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_param_1];
+; SM107-NEXT: sub.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; SM107-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM107-NEXT: ret;
+;
+; SM100-LABEL: sub_f32x2_f16x2(
+; SM100: {
+; SM100-NEXT: .reg .b16 %rs<3>;
+; SM100-NEXT: .reg .b32 %r<5>;
+; SM100-NEXT: .reg .b64 %rd<4>;
+; SM100-EMPTY:
+; SM100-NEXT: // %bb.0:
+; SM100-NEXT: ld.param::func.b32 %r1, [sub_f32x2_f16x2_param_0];
+; SM100-NEXT: mul.rn.f16x2 %r2, %r1, %r1;
+; SM100-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_param_1];
+; SM100-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM100-NEXT: cvt.f32.f16 %r3, %rs2;
+; SM100-NEXT: cvt.f32.f16 %r4, %rs1;
+; SM100-NEXT: mov.b64 %rd2, {%r4, %r3};
+; SM100-NEXT: sub.rn.f32x2 %rd3, %rd2, %rd1;
+; SM100-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM100-NEXT: ret;
+ %x = fmul <2 x half> %a, %a
+ %e = fpext <2 x half> %x to <2 x float>
+ %nb = fneg <2 x float> %b
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_bf16x2(<2 x bfloat> %a, <2 x float> %b) {
+; SM107-LABEL: sub_f32x2_bf16x2(
+; SM107: {
+; SM107-NEXT: .reg .b32 %r<3>;
+; SM107-NEXT: .reg .b64 %rd<3>;
+; SM107-EMPTY:
+; SM107-NEXT: // %bb.0:
+; SM107-NEXT: ld.param::func.b32 %r1, [sub_f32x2_bf16x2_param_0];
+; SM107-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; SM107-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_param_1];
+; SM107-NEXT: sub.rn.f32x2.bf16x2.f32x2 %rd2, %r2, %rd1;
+; SM107-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; SM107-NEXT: ret;
+;
+; SM100-LABEL: sub_f32x2_bf16x2(
+; SM100: {
+; SM100-NEXT: .reg .b16 %rs<3>;
+; SM100-NEXT: .reg .b32 %r<5>;
+; SM100-NEXT: .reg .b64 %rd<4>;
+; SM100-EMPTY:
+; SM100-NEXT: // %bb.0:
+; SM100-NEXT: ld.param::func.b32 %r1, [sub_f32x2_bf16x2_param_0];
+; SM100-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; SM100-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_param_1];
+; SM100-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; SM100-NEXT: cvt.f32.bf16 %r3, %rs2;
+; SM100-NEXT: cvt.f32.bf16 %r4, %rs1;
+; SM100-NEXT: mov.b64 %rd2, {%r4, %r3};
+; SM100-NEXT: sub.rn.f32x2 %rd3, %rd2, %rd1;
+; SM100-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; SM100-NEXT: ret;
+ %x = fmul <2 x bfloat> %a, %a
+ %e = fpext <2 x bfloat> %x to <2 x float>
+ %nb = fneg <2 x float> %b
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+ ret <2 x float> %r
+}
+
+;
+; Downconverting
+;
+
+define <2 x half> @add_rz_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; SM107-LABEL: add_rz_f16x2_f32x2(
+; SM107: {
+; SM107-NEXT: .reg .b32 %r<2>;
+; SM107-NEXT: .reg .b64 %rd<3>;
+; SM107-EMPTY:
+; SM107-NEXT: // %bb.0:
+; SM107-NEXT: ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_param_0];
+; SM107-NEXT: ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_param_1];
+; SM107-NEXT: add.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; SM107-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM107-NEXT: ret;
+;
+; SM100-LABEL: add_rz_f16x2_f32x2(
+; SM100: {
+; SM100-NEXT: .reg .b32 %r<4>;
+; SM100-NEXT: .reg .b64 %rd<4>;
+; SM100-EMPTY:
+; SM100-NEXT: // %bb.0:
+; SM100-NEXT: ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_param_0];
+; SM100-NEXT: ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_param_1];
+; SM100-NEXT: add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; SM100-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; SM100-NEXT: cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; SM100-NEXT: st.param::func.b32 [func_retval0], %r3;
+; SM100-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; SM107-LABEL: add_rz_bf16x2_f32x2(
+; SM107: {
+; SM107-NEXT: .reg .b32 %r<2>;
+; SM107-NEXT: .reg .b64 %rd<3>;
+; SM107-EMPTY:
+; SM107-NEXT: // %bb.0:
+; SM107-NEXT: ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_param_0];
+; SM107-NEXT: ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_param_1];
+; SM107-NEXT: add.rz.bf16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; SM107-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM107-NEXT: ret;
+;
+; SM100-LABEL: add_rz_bf16x2_f32x2(
+; SM100: {
+; SM100-NEXT: .reg .b32 %r<4>;
+; SM100-NEXT: .reg .b64 %rd<4>;
+; SM100-EMPTY:
+; SM100-NEXT: // %bb.0:
+; SM100-NEXT: ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_param_0];
+; SM100-NEXT: ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_param_1];
+; SM100-NEXT: add.rz.f32x2 %rd3, %rd1, %rd2;
+; SM100-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; SM100-NEXT: cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; SM100-NEXT: st.param::func.b32 [func_retval0], %r3;
+; SM100-NEXT: ret;
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x bfloat> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; SM107-LABEL: sub_rz_f16x2_f32x2(
+; SM107: {
+; SM107-NEXT: .reg .b32 %r<2>;
+; SM107-NEXT: .reg .b64 %rd<3>;
+; SM107-EMPTY:
+; SM107-NEXT: // %bb.0:
+; SM107-NEXT: ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_param_0];
+; SM107-NEXT: ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_param_1];
+; SM107-NEXT: sub.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; SM107-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM107-NEXT: ret;
+;
+; SM100-LABEL: sub_rz_f16x2_f32x2(
+; SM100: {
+; SM100-NEXT: .reg .b32 %r<4>;
+; SM100-NEXT: .reg .b64 %rd<4>;
+; SM100-EMPTY:
+; SM100-NEXT: // %bb.0:
+; SM100-NEXT: ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_param_0];
+; SM100-NEXT: ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_param_1];
+; SM100-NEXT: sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; SM100-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; SM100-NEXT: cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; SM100-NEXT: st.param::func.b32 [func_retval0], %r3;
+; SM100-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; SM107-LABEL: sub_rz_bf16x2_f32x2(
+; SM107: {
+; SM107-NEXT: .reg .b32 %r<2>;
+; SM107-NEXT: .reg .b64 %rd<3>;
+; SM107-EMPTY:
+; SM107-NEXT: // %bb.0:
+; SM107-NEXT: ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_param_0];
+; SM107-NEXT: ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_param_1];
+; SM107-NEXT: sub.rz.bf16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; SM107-NEXT: st.param::func.b32 [func_retval0], %r1;
+; SM107-NEXT: ret;
+;
+; SM100-LABEL: sub_rz_bf16x2_f32x2(
+; SM100: {
+; SM100-NEXT: .reg .b32 %r<4>;
+; SM100-NEXT: .reg .b64 %rd<4>;
+; SM100-EMPTY:
+; SM100-NEXT: // %bb.0:
+; SM100-NEXT: ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_param_0];
+; SM100-NEXT: ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_param_1];
+; SM100-NEXT: sub.rz.f32x2 %rd3, %rd1, %rd2;
+; SM100-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; SM100-NEXT: cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; SM100-NEXT: st.param::func.b32 [func_retval0], %r3;
+; SM100-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x bfloat> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert-invalid.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert-invalid.ll
new file mode 100644
index 0000000000000..ac9f35a08f4fd
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert-invalid.ll
@@ -0,0 +1,274 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+;
+; Cases where the mixed-precision instruction must not be selected.
+;
+
+;
+; F16x2
+;
+
+define <2 x half> @sub_rz_f16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_extra_use(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_extra_use_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_extra_use_param_1];
+; CHECK-NEXT: sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: ld.param::func.b64 %rd4, [sub_rz_f16x2_f32x2_extra_use_param_2];
+; CHECK-NEXT: st.b64 [%rd4], %rd3;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ ; extra use
+ store <2 x float> %sum, ptr %p
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+; unsupported forms
+
+define <2 x half> @sub_rz_f16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_swapped_halves(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_swapped_halves_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_swapped_halves_param_1];
+; CHECK-NEXT: sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.f16x2.f32 %r3, %r1, %r2;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %lo, float %hi, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_no_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_no_ftz(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_no_ftz_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_no_ftz_param_1];
+; CHECK-NEXT: sub.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x half> @sub_rn_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rn_f16x2_f32x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rn_f16x2_f32x2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rn_f16x2_f32x2_param_1];
+; CHECK-NEXT: sub.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 1)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_rn_convert(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_rn_convert(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_rn_convert_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_rn_convert_param_1];
+; CHECK-NEXT: sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rn.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_relu(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_relu(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_relu_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_relu_param_1];
+; CHECK-NEXT: sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.relu.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_distinct_subs(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_distinct_subs(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_distinct_subs_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_distinct_subs_param_1];
+; CHECK-NEXT: ld.param::func.b64 %rd3, [sub_rz_f16x2_f32x2_distinct_subs_param_2];
+; CHECK-NEXT: sub.rz.ftz.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NEXT: sub.rz.ftz.f32x2 %rd5, %rd1, %rd3;
+; CHECK-NEXT: mov.b64 {%r1, _}, %rd4;
+; CHECK-NEXT: mov.b64 {_, %r2}, %rd5;
+; CHECK-NEXT: cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %nc = fneg <2 x float> %c
+ %sum1 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ %sum2 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nc, i32 0)
+ %lo = extractelement <2 x float> %sum1, i32 0
+ %hi = extractelement <2 x float> %sum2, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+;
+; BF16x2
+;
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2_extra_use(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_extra_use_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_extra_use_param_1];
+; CHECK-NEXT: sub.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: ld.param::func.b64 %rd4, [sub_rz_bf16x2_f32x2_extra_use_param_2];
+; CHECK-NEXT: st.b64 [%rd4], %rd3;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ ; extra use
+ store <2 x float> %sum, ptr %p
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x bfloat> %r
+}
+
+; unsupported forms
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2_swapped_halves(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_swapped_halves_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_swapped_halves_param_1];
+; CHECK-NEXT: sub.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.bf16x2.f32 %r3, %r1, %r2;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %lo, float %hi, i1 false)
+ ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2_ftz(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_ftz_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_ftz_param_1];
+; CHECK-NEXT: sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @sub_rn_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rn_bf16x2_f32x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<4>;
+; CHECK-NEXT: .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rn_bf16x2_f32x2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rn_bf16x2_f32x2_param_1];
+; CHECK-NEXT: sub.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT: mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT: cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 1)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x bfloat> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert.ll
new file mode 100644
index 0000000000000..246b04173b657
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert.ll
@@ -0,0 +1,91 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+;
+; F16x2
+;
+
+define <2 x half> @sub_rz_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_param_1];
+; CHECK-NEXT: sub.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_negated_minuend(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_negated_minuend(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_negated_minuend_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_negated_minuend_param_1];
+; CHECK-NEXT: sub.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd2, %rd1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %na = fneg <2 x float> %a
+ %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %na, <2 x float> %b, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x half> %r
+}
+
+;
+; BF16x2
+;
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_param_1];
+; CHECK-NEXT: sub.rz.bf16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %nb = fneg <2 x float> %b
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2_negated_minuend(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2_negated_minuend(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<2>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_negated_minuend_param_0];
+; CHECK-NEXT: ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_negated_minuend_param_1];
+; CHECK-NEXT: sub.rz.bf16x2.f32x2.f32x2 %r1, %rd2, %rd1;
+; CHECK-NEXT: st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT: ret;
+ %na = fneg <2 x float> %a
+ %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %na, <2 x float> %b, i32 0)
+ %lo = extractelement <2 x float> %sum, i32 0
+ %hi = extractelement <2 x float> %sum, i32 1
+ %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+ ret <2 x bfloat> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert-invalid.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert-invalid.ll
new file mode 100644
index 0000000000000..6b51930875a78
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert-invalid.ll
@@ -0,0 +1,539 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck --check-prefixes=CHECK-NOFTZ %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | FileCheck --check-prefixes=CHECK-FTZ %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | %ptxas-verify -arch=sm_107f %}
+
+;
+; Cases where the mixed-precision instruction must not be selected.
+;
+
+;
+; F16x2
+;
+
+define <2 x float> @sub_f32x2_f16x2_extra_use(<2 x half> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_extra_use(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_extra_use_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_extra_use_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd3, [sub_f32x2_f16x2_extra_use_param_2];
+; CHECK-NOFTZ-NEXT: st.b64 [%rd3], %rd1;
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_extra_use(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_extra_use_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_extra_use_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd3, [sub_f32x2_f16x2_extra_use_param_2];
+; CHECK-FTZ-NEXT: st.b64 [%rd3], %rd1;
+; CHECK-FTZ-NEXT: sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ ; extra use
+ store <2 x float> %e, ptr %p
+ %nb = fneg <2 x float> %b
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_f16x2_generic_lanes_multiuse(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_lanes_multiuse(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_lanes_multiuse_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_lanes_multiuse_param_1];
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd4, %rd1, %rd3;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_lanes_multiuse(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_lanes_multiuse_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_lanes_multiuse_param_1];
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd4, %rd1, %rd3;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ %r1 = fsub <2 x float> %e, %b
+ ; extra use
+ %r2 = fsub <2 x float> %e, %r1
+ ret <2 x float> %r2
+}
+
+define <2 x float> @sub_f32x2_f16x2_generic_packed_multiuse(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_packed_multiuse(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [sub_f32x2_f16x2_generic_packed_multiuse_param_0];
+; CHECK-NOFTZ-NEXT: mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_packed_multiuse_param_1];
+; CHECK-NOFTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r3, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r4, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd2, {%r4, %r3};
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd4, %rd2, %rd3;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_packed_multiuse(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [sub_f32x2_f16x2_generic_packed_multiuse_param_0];
+; CHECK-FTZ-NEXT: mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_packed_multiuse_param_1];
+; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r3, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r4, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd4, %rd2, %rd3;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %x = fmul <2 x half> %a, %a
+ %e = fpext <2 x half> %x to <2 x float>
+ %r1 = fsub <2 x float> %e, %b
+ ; extra use
+ %r2 = fsub <2 x float> %e, %r1
+ ret <2 x float> %r2
+}
+
+define <2 x float> @sub_f32x2_f16x2_generic_both_lanes_extra_use(<2 x half> %a, <2 x float> %b, ptr %p, ptr %q) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_both_lanes_extra_use(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd3, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r2, %rs2;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd4, {%r1, %r2};
+; CHECK-NOFTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT: st.b32 [%rd3], %r2;
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd5, %rd4, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_both_lanes_extra_use(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd3, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs2;
+; CHECK-FTZ-NEXT: mov.b64 %rd4, {%r1, %r2};
+; CHECK-FTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT: st.b32 [%rd3], %r2;
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd5, %rd4, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT: ret;
+ %a0 = extractelement <2 x half> %a, i32 0
+ %a1 = extractelement <2 x half> %a, i32 1
+ %e0 = fpext half %a0 to float
+ %e1 = fpext half %a1 to float
+ %v0 = insertelement <2 x float> poison, float %e0, i32 0
+ %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+ ; extra uses
+ store float %e0, ptr %p
+ store float %e1, ptr %q
+ %r = fsub <2 x float> %v1, %b
+ ret <2 x float> %r
+}
+
+; unsupported forms
+
+define <2 x float> @sub_f32x2_f16x2_reversed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_reversed(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_reversed_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_reversed_param_1];
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_reversed(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_reversed_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_reversed_param_1];
+; CHECK-FTZ-NEXT: sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ %ne = fneg <2 x float> %e
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %ne, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_f16x2_generic_reversed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_reversed(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_reversed_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_reversed_param_1];
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_reversed(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_reversed_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_reversed_param_1];
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ %r = fsub <2 x float> %b, %e
+ ret <2 x float> %r
+}
+
+;
+; BF16x2
+;
+
+define <2 x float> @sub_f32x2_bf16x2_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_extra_use(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_extra_use_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_extra_use_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd3, [sub_f32x2_bf16x2_extra_use_param_2];
+; CHECK-NOFTZ-NEXT: st.b64 [%rd3], %rd1;
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_extra_use(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_extra_use_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_extra_use_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd3, [sub_f32x2_bf16x2_extra_use_param_2];
+; CHECK-FTZ-NEXT: st.b64 [%rd3], %rd1;
+; CHECK-FTZ-NEXT: sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ ; extra use
+ store <2 x float> %e, ptr %p
+ %nb = fneg <2 x float> %b
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_bf16x2_generic_lanes_multiuse(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_lanes_multiuse(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_lanes_multiuse_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_lanes_multiuse_param_1];
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd4, %rd1, %rd3;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_lanes_multiuse(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_lanes_multiuse_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_lanes_multiuse_param_1];
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd4, %rd1, %rd3;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ %r1 = fsub <2 x float> %e, %b
+ ; extra use
+ %r2 = fsub <2 x float> %e, %r1
+ ret <2 x float> %r2
+}
+
+define <2 x float> @sub_f32x2_bf16x2_generic_packed_multiuse(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_packed_multiuse(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [sub_f32x2_bf16x2_generic_packed_multiuse_param_0];
+; CHECK-NOFTZ-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_packed_multiuse_param_1];
+; CHECK-NOFTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r3, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r4, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd2, {%r4, %r3};
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd4, %rd2, %rd3;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_packed_multiuse(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [sub_f32x2_bf16x2_generic_packed_multiuse_param_0];
+; CHECK-FTZ-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_packed_multiuse_param_1];
+; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r3, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r4, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd4, %rd2, %rd3;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %x = fmul <2 x bfloat> %a, %a
+ %e = fpext <2 x bfloat> %x to <2 x float>
+ %r1 = fsub <2 x float> %e, %b
+ ; extra use
+ %r2 = fsub <2 x float> %e, %r1
+ ret <2 x float> %r2
+}
+
+define <2 x float> @sub_f32x2_bf16x2_generic_both_lanes_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p, ptr %q) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_both_lanes_extra_use(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd3, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r2, %rs2;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd4, {%r1, %r2};
+; CHECK-NOFTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT: st.b32 [%rd3], %r2;
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd5, %rd4, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_both_lanes_extra_use(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd3, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs2;
+; CHECK-FTZ-NEXT: mov.b64 %rd4, {%r1, %r2};
+; CHECK-FTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT: st.b32 [%rd3], %r2;
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd5, %rd4, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT: ret;
+ %a0 = extractelement <2 x bfloat> %a, i32 0
+ %a1 = extractelement <2 x bfloat> %a, i32 1
+ %e0 = fpext bfloat %a0 to float
+ %e1 = fpext bfloat %a1 to float
+ %v0 = insertelement <2 x float> poison, float %e0, i32 0
+ %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+ ; extra uses
+ store float %e0, ptr %p
+ store float %e1, ptr %q
+ %r = fsub <2 x float> %v1, %b
+ ret <2 x float> %r
+}
+
+; unsupported forms
+
+define <2 x float> @sub_f32x2_bf16x2_reversed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_reversed(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_reversed_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_reversed_param_1];
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_reversed(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_reversed_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_reversed_param_1];
+; CHECK-FTZ-NEXT: sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ %ne = fneg <2 x float> %e
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %ne, <2 x float> %b, i32 1)
+ ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_bf16x2_generic_reversed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_reversed(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_reversed_param_0];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_reversed_param_1];
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_reversed(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_reversed_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_reversed_param_1];
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ %r = fsub <2 x float> %b, %e
+ ret <2 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert.ll
new file mode 100644
index 0000000000000..a2f0df199e12d
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert.ll
@@ -0,0 +1,394 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck --check-prefixes=CHECK,CHECK-NOFTZ %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | FileCheck --check-prefixes=CHECK,CHECK-FTZ %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | %ptxas-verify -arch=sm_107f %}
+
+;
+; F16x2
+;
+
+; lanes
+
+define <2 x float> @sub_f32x2_f16x2_lanes(<2 x half> %a0, <2 x half> %a1, <2 x half> %a2, <2 x half> %a3, <2 x float> %b) {
+; CHECK-LABEL: sub_f32x2_f16x2_lanes(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<9>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_lanes_param_0];
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs3, %rs4}, [sub_f32x2_f16x2_lanes_param_1];
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs5, %rs6}, [sub_f32x2_f16x2_lanes_param_2];
+; CHECK-NEXT: mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_lanes_param_4];
+; CHECK-NEXT: sub.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs7, %rs8}, [sub_f32x2_f16x2_lanes_param_3];
+; CHECK-NEXT: mov.b32 %r2, {%rs3, %rs4};
+; CHECK-NEXT: sub.rz.f32x2.f16x2.f32x2 %rd3, %r2, %rd2;
+; CHECK-NEXT: mov.b32 %r3, {%rs5, %rs6};
+; CHECK-NEXT: sub.rm.f32x2.f16x2.f32x2 %rd4, %r3, %rd3;
+; CHECK-NEXT: mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT: sub.rp.f32x2.f16x2.f32x2 %rd5, %r4, %rd4;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NEXT: ret;
+ %e0 = fpext <2 x half> %a0 to <2 x float>
+ %n0 = fneg <2 x float> %b
+ %r0 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e0, <2 x float> %n0, i32 1)
+ %e1 = fpext <2 x half> %a1 to <2 x float>
+ %n1 = fneg <2 x float> %r0
+ %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e1, <2 x float> %n1, i32 0)
+ %e2 = fpext <2 x half> %a2 to <2 x float>
+ %n2 = fneg <2 x float> %r1
+ %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e2, <2 x float> %n2, i32 3)
+ %e3 = fpext <2 x half> %a3 to <2 x float>
+ %n3 = fneg <2 x float> %r2
+ %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e3, <2 x float> %n3, i32 2)
+ ret <2 x float> %r3
+}
+
+; packed
+
+define <2 x float> @sub_f32x2_f16x2_packed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_packed(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [sub_f32x2_f16x2_packed_param_0];
+; CHECK-NOFTZ-NEXT: mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_packed_param_1];
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_packed(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [sub_f32x2_f16x2_packed_param_0];
+; CHECK-FTZ-NEXT: mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_packed_param_1];
+; CHECK-FTZ-NEXT: sub.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-FTZ-NEXT: ret;
+ %x = fmul <2 x half> %a, %a
+ %e = fpext <2 x half> %x to <2 x float>
+ %nb = fneg <2 x float> %b
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+ ret <2 x float> %r
+}
+
+; generic fsub
+
+define <2 x float> @sub_f32x2_f16x2_generic_lanes(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_lanes(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<2>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_lanes_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_lanes_param_1];
+; CHECK-NOFTZ-NEXT: mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_lanes(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_lanes_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_lanes_param_1];
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x half> %a to <2 x float>
+ %r = fsub <2 x float> %e, %b
+ ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_f16x2_generic_packed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_packed(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [sub_f32x2_f16x2_generic_packed_param_0];
+; CHECK-NOFTZ-NEXT: mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_packed_param_1];
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_packed(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [sub_f32x2_f16x2_generic_packed_param_0];
+; CHECK-FTZ-NEXT: mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_packed_param_1];
+; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r3, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r4, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %x = fmul <2 x half> %a, %a
+ %e = fpext <2 x half> %x to <2 x float>
+ %r = fsub <2 x float> %e, %b
+ ret <2 x float> %r
+}
+
+; generic fsub - partially shared extension
+
+define <2 x float> @sub_f32x2_f16x2_generic_lane_extra_use(<2 x half> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_lane_extra_use(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_lane_extra_use_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_lane_extra_use_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_lane_extra_use_param_2];
+; CHECK-NOFTZ-NEXT: cvt.f32.f16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT: mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2.f16x2.f32x2 %rd3, %r2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_lane_extra_use(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_lane_extra_use_param_0];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_lane_extra_use_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_lane_extra_use_param_2];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r1, %rs1;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.f16 %r2, %rs2;
+; CHECK-FTZ-NEXT: mov.b64 %rd3, {%r1, %r2};
+; CHECK-FTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd4, %rd3, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %a0 = extractelement <2 x half> %a, i32 0
+ %a1 = extractelement <2 x half> %a, i32 1
+ %e0 = fpext half %a0 to float
+ %e1 = fpext half %a1 to float
+ %v0 = insertelement <2 x float> poison, float %e0, i32 0
+ %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+ store float %e0, ptr %p
+ %r = fsub <2 x float> %v1, %b
+ ret <2 x float> %r
+}
+
+;
+; BF16x2
+;
+
+; lanes
+
+define <2 x float> @sub_f32x2_bf16x2_lanes(<2 x bfloat> %a0, <2 x bfloat> %a1, <2 x bfloat> %a2, <2 x bfloat> %a3, <2 x float> %b) {
+; CHECK-LABEL: sub_f32x2_bf16x2_lanes(
+; CHECK: {
+; CHECK-NEXT: .reg .b16 %rs<9>;
+; CHECK-NEXT: .reg .b32 %r<5>;
+; CHECK-NEXT: .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_lanes_param_0];
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs3, %rs4}, [sub_f32x2_bf16x2_lanes_param_1];
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs5, %rs6}, [sub_f32x2_bf16x2_lanes_param_2];
+; CHECK-NEXT: mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_lanes_param_4];
+; CHECK-NEXT: sub.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NEXT: ld.param::func.v2.b16 {%rs7, %rs8}, [sub_f32x2_bf16x2_lanes_param_3];
+; CHECK-NEXT: mov.b32 %r2, {%rs3, %rs4};
+; CHECK-NEXT: sub.rz.f32x2.bf16x2.f32x2 %rd3, %r2, %rd2;
+; CHECK-NEXT: mov.b32 %r3, {%rs5, %rs6};
+; CHECK-NEXT: sub.rm.f32x2.bf16x2.f32x2 %rd4, %r3, %rd3;
+; CHECK-NEXT: mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT: sub.rp.f32x2.bf16x2.f32x2 %rd5, %r4, %rd4;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NEXT: ret;
+ %e0 = fpext <2 x bfloat> %a0 to <2 x float>
+ %n0 = fneg <2 x float> %b
+ %r0 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e0, <2 x float> %n0, i32 1)
+ %e1 = fpext <2 x bfloat> %a1 to <2 x float>
+ %n1 = fneg <2 x float> %r0
+ %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e1, <2 x float> %n1, i32 0)
+ %e2 = fpext <2 x bfloat> %a2 to <2 x float>
+ %n2 = fneg <2 x float> %r1
+ %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e2, <2 x float> %n2, i32 3)
+ %e3 = fpext <2 x bfloat> %a3 to <2 x float>
+ %n3 = fneg <2 x float> %r2
+ %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e3, <2 x float> %n3, i32 2)
+ ret <2 x float> %r3
+}
+
+; packed
+
+define <2 x float> @sub_f32x2_bf16x2_packed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_f32x2_bf16x2_packed(
+; CHECK: {
+; CHECK-NEXT: .reg .b32 %r<3>;
+; CHECK-NEXT: .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT: // %bb.0:
+; CHECK-NEXT: ld.param::func.b32 %r1, [sub_f32x2_bf16x2_packed_param_0];
+; CHECK-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_packed_param_1];
+; CHECK-NEXT: sub.rn.f32x2.bf16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NEXT: ret;
+ %x = fmul <2 x bfloat> %a, %a
+ %e = fpext <2 x bfloat> %x to <2 x float>
+ %nb = fneg <2 x float> %b
+ %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+ ret <2 x float> %r
+}
+
+; generic fsub
+
+define <2 x float> @sub_f32x2_bf16x2_generic_lanes(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_lanes(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<2>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_lanes_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_lanes_param_1];
+; CHECK-NOFTZ-NEXT: mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_lanes(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_lanes_param_0];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_lanes_param_1];
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %e = fpext <2 x bfloat> %a to <2 x float>
+ %r = fsub <2 x float> %e, %b
+ ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_bf16x2_generic_packed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_packed(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.b32 %r1, [sub_f32x2_bf16x2_generic_packed_param_0];
+; CHECK-NOFTZ-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_packed_param_1];
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2.bf16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_packed(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.b32 %r1, [sub_f32x2_bf16x2_generic_packed_param_0];
+; CHECK-FTZ-NEXT: mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_packed_param_1];
+; CHECK-FTZ-NEXT: mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r3, %rs2;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r4, %rs1;
+; CHECK-FTZ-NEXT: mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT: ret;
+ %x = fmul <2 x bfloat> %a, %a
+ %e = fpext <2 x bfloat> %x to <2 x float>
+ %r = fsub <2 x float> %e, %b
+ ret <2 x float> %r
+}
+
+; generic fsub - partially shared extension
+
+define <2 x float> @sub_f32x2_bf16x2_generic_lane_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_lane_extra_use(
+; CHECK-NOFTZ: {
+; CHECK-NOFTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT: .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT: // %bb.0:
+; CHECK-NOFTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_lane_extra_use_param_0];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_lane_extra_use_param_1];
+; CHECK-NOFTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_lane_extra_use_param_2];
+; CHECK-NOFTZ-NEXT: cvt.f32.bf16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT: mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT: sub.rn.f32x2.bf16x2.f32x2 %rd3, %r2, %rd1;
+; CHECK-NOFTZ-NEXT: st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT: ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_lane_extra_use(
+; CHECK-FTZ: {
+; CHECK-FTZ-NEXT: .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT: .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT: .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT: // %bb.0:
+; CHECK-FTZ-NEXT: ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_lane_extra_use_param_0];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_lane_extra_use_param_1];
+; CHECK-FTZ-NEXT: ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_lane_extra_use_param_2];
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r1, %rs1;
+; CHECK-FTZ-NEXT: cvt.ftz.f32.bf16 %r2, %rs2;
+; CHECK-FTZ-NEXT: mov.b64 %rd3, {%r1, %r2};
+; CHECK-FTZ-NEXT: st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT: sub.rn.ftz.f32x2 %rd4, %rd3, %rd1;
+; CHECK-FTZ-NEXT: st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT: ret;
+ %a0 = extractelement <2 x bfloat> %a, i32 0
+ %a1 = extractelement <2 x bfloat> %a, i32 1
+ %e0 = fpext bfloat %a0 to float
+ %e1 = fpext bfloat %a1 to float
+ %v0 = insertelement <2 x float> poison, float %e0, i32 0
+ %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+ store float %e0, ptr %p
+ %r = fsub <2 x float> %v1, %b
+ ret <2 x float> %r
+}
More information about the llvm-commits
mailing list