[llvm] [NVPTX] Add support for f32x2 mixed-precision add/sub (PR #221957)

Wed Sep 9 22:46:25 PDT 2026

https://github.com/Wolfram70 updated https://github.com/llvm/llvm-project/pull/221957

>From cc5c3eb6149350ac83711ebbecad20b9dace509e Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Mon, 7 Sep 2026 14:39:58 +0000
Subject: [PATCH 1/7] [NVPTX] Add support for f32x2 mixed-precision additions

This change adds support for mixed precision addition of `f16x2`
and `bf16x2` with `f32x2`, where the following upconverting
patterns:
```
%e = fpext <2 x half> %h to <2 x float>
%res = fp-operation(%e, ...)
...
%e = fpext <2 x bfloat> %b to <2 x float>
%res = fp-operation(%e, ...)

where the fp-operation can be any of:
- fadd
- llvm.nvvm.fadd.v2f32
```
are lowered to `add.{rnd}.f32x2.{f16x2/bf16x2}.f32x2`, and the
following downconverting pattern:
```
%sum = llvm.nvvm.fadd{.ftz}.v2f32(%a, %b, rz)
%lo = extractelement <2 x float> %sum, i32 0
%hi = extractelement <2 x float> %sum, i32 1
%res = llvm.nvvm.ff2{f16x2/bf16x2}.rz(%hi, %lo, false)
```
is lowered to `add.rz{.ftz}.{f16x2/bf16x2}.f32x2.f32x2`. These
instructions combine the conversion and the addition into one
instruction from `sm_107f` onwards.

The tests have been verified through ptxas-13.4.

PTX spec reference:
https://docs.nvidia.com/cuda/parallel-thread-execution/#mixed-precision-floating-point-instructions
---
 llvm/lib/Target/NVPTX/NVPTXIntrinsics.td      |  61 +++
 .../mixed-precision-add-f32x2-downconvert.ll  | 330 +++++++++++
 .../mixed-precision-add-f32x2-upconvert.ll    | 517 ++++++++++++++++++
 3 files changed, 908 insertions(+)
 create mode 100644 llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll

diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 220ef64732830..c047c4fd4f94b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2495,6 +2495,19 @@ def INT_NVVM_ADD_D :
   F_MATH_2_RNDOP_TY<"add.${rnd}.f64", F64RT, int_nvvm_fadd>;
 
 // mixed precision
+
+// FP_EXTEND to v2f32 is scalarized before isel, leaving one of two shapes
+// behind depending on whether the source lanes share a register.
+def fpextend_v2f32_from_packed : PatFrag<(ops node:$a),
+                                         (v2f32 (build_vector
+                                           (f32 (fpextend (extractelt node:$a, 0))),
+                                           (f32 (fpextend (extractelt node:$a, 1)))))>;
+
+def fpextend_v2f32_from_lanes : PatFrag<(ops node:$a0, node:$a1),
+                                        (v2f32 (build_vector
+                                          (f32 (fpextend node:$a0)),
+                                          (f32 (fpextend node:$a1))))>;
+
 foreach rnd = FPRoundingModes in {
   defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
 
@@ -2508,6 +2521,41 @@ foreach rnd = FPRoundingModes in {
              (f32 (fpextend type:$a)),
              f32:$b, rnd_imm))]>,
         Requires<[SM100]>;
+
+  foreach t = [F16X2RT, BF16X2RT] in {
+    def INT_NVVM_MIXED_ADD_ # rnd # _f32x2_ # t.PtxType :
+      BasicNVPTXInst<(outs B64:$dst), (ins t.RC:$a, B64:$b),
+        StrJoin<".", ["add", rnd, "f32x2", t.PtxType, "f32x2"]>.ret,
+        [(set v2f32:$dst,
+          (int_nvvm_fadd (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b,
+                         rnd_imm))]>,
+      Requires<[SM107f]>;
+
+    def : Pat<(v2f32 (int_nvvm_fadd
+                       (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
+                                                  t.Ty.ElementType:$a1),
+                       v2f32:$b, rnd_imm)),
+              (!cast<Instruction>("INT_NVVM_MIXED_ADD_" # rnd # "_f32x2_" #
+                                  t.PtxType) (t.Ty (V2I16toI32 $a0, $a1)), $b)>,
+      Requires<[SM107f]>;
+  }
+}
+
+// mixed precision, downconverting
+foreach t = [F16X2RT, BF16X2RT] in {
+  defvar ftz = !if(!eq(t, F16X2RT), "ftz", "");
+  defvar AddOp = !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", ftz]>.ret);
+  defvar CvtOp = !cast<Intrinsic>("int_nvvm_ff2" # t.PtxType # "_rz");
+
+  def INT_NVVM_MIXED_ADD_rz_ # t.PtxType # _f32x2 :
+    BasicNVPTXInst<(outs t.RC:$dst), (ins B64:$a, B64:$b),
+      StrJoin<".", ["add.rz", ftz, t.PtxType, "f32x2", "f32x2"]>.ret,
+      // The cvt packs its first argument into the high half of the result.
+      [(set t.Ty:$dst,
+        (CvtOp (extractelt (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 1),
+               (extractelt (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 0),
+               /*pzo=*/0))]>,
+    Requires<[SM107f]>;
 }
 
 // Pattern for fadd when there is no FTZ flag
@@ -2518,6 +2566,19 @@ let Predicates = [SM100, doNoF32FTZ] in {
             (INT_NVVM_MIXED_ADD_rn_f32_bf16 B16:$a, B32:$b)>;
 }
 
+let Predicates = [SM107f, doNoF32FTZ] in
+  foreach t = [F16X2RT, BF16X2RT] in {
+    defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_ADD_rn_f32x2_" # t.PtxType);
+
+    def : Pat<(v2f32 (fadd (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b)),
+              (Inst $a, $b)>;
+
+    def : Pat<(v2f32 (fadd (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
+                                                      t.Ty.ElementType:$a1),
+                           v2f32:$b)),
+              (Inst (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
+  }
+
 //
 // Sub
 //
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert.ll
new file mode 100644
index 0000000000000..9dbfe1e3ab014
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert.ll
@@ -0,0 +1,330 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+;
+;  F16x2
+;
+
+define <2 x half> @add_rz_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_param_1];
+; CHECK-NEXT:    add.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_commuted(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_commuted(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_commuted_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_commuted_param_1];
+; CHECK-NEXT:    add.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd2, %rd1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %b, <2 x float> %a, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_extra_use(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_extra_use_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_extra_use_param_1];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    ld.param::func.b64 %rd4, [add_rz_f16x2_f32x2_extra_use_param_2];
+; CHECK-NEXT:    st.b64 [%rd4], %rd3;
+; CHECK-NEXT:    add.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  store <2 x float> %sum, ptr %p
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+; invalid patterns
+
+define <2 x half> @add_rz_f16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_swapped_halves(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_swapped_halves_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_swapped_halves_param_1];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %lo, float %hi, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_no_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_no_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_no_ftz_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_no_ftz_param_1];
+; CHECK-NEXT:    add.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @add_rn_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rn_f16x2_f32x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rn_f16x2_f32x2_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rn_f16x2_f32x2_param_1];
+; CHECK-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_rn_convert(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_rn_convert(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_rn_convert_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_rn_convert_param_1];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rn.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_relu(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_relu(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_relu_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_relu_param_1];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.relu.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_distinct_adds(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_distinct_adds(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_distinct_adds_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_distinct_adds_param_1];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    ld.param::func.b64 %rd4, [add_rz_f16x2_f32x2_distinct_adds_param_2];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd5, %rd1, %rd4;
+; CHECK-NEXT:    mov.b64 {%r1, _}, %rd3;
+; CHECK-NEXT:    mov.b64 {_, %r2}, %rd5;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum1 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %sum2 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %c, i32 0)
+  %lo = extractelement <2 x float> %sum1, i32 0
+  %hi = extractelement <2 x float> %sum2, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+;
+;  BF16x2
+;
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_param_1];
+; CHECK-NEXT:    add.rz.bf16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2_commuted(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2_commuted(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_commuted_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_commuted_param_1];
+; CHECK-NEXT:    add.rz.bf16x2.f32x2.f32x2 %r1, %rd2, %rd1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %b, <2 x float> %a, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2_extra_use(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_extra_use_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_extra_use_param_1];
+; CHECK-NEXT:    add.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    ld.param::func.b64 %rd4, [add_rz_bf16x2_f32x2_extra_use_param_2];
+; CHECK-NEXT:    st.b64 [%rd4], %rd3;
+; CHECK-NEXT:    add.rz.bf16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  store <2 x float> %sum, ptr %p
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
+
+; invalid patterns
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2_swapped_halves(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_swapped_halves_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_swapped_halves_param_1];
+; CHECK-NEXT:    add.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %lo, float %hi, i1 false)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_ftz_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_ftz_param_1];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @add_rn_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rn_bf16x2_f32x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rn_bf16x2_f32x2_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rn_bf16x2_f32x2_param_1];
+; CHECK-NEXT:    add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll
new file mode 100644
index 0000000000000..35d53eebf8ef0
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll
@@ -0,0 +1,517 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck --check-prefixes=CHECK,CHECK-NOFTZ %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | FileCheck --check-prefixes=CHECK,CHECK-FTZ %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | %ptxas-verify -arch=sm_107f %}
+
+;
+;  F16x2
+;
+
+; lanes
+
+define <2 x float> @add_f32x2_f16x2_lanes(<2 x half> %a, <2 x float> %b) {
+; CHECK-LABEL: add_f32x2_f16x2_lanes(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_lanes_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_lanes_param_1];
+; CHECK-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NEXT:    add.rz.f32x2.f16x2.f32x2 %rd3, %r1, %rd2;
+; CHECK-NEXT:    add.rm.f32x2.f16x2.f32x2 %rd4, %r1, %rd3;
+; CHECK-NEXT:    add.rp.f32x2.f16x2.f32x2 %rd5, %r1, %rd4;
+; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %r1, i32 0)
+  %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %r2, i32 3)
+  %r4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %r3, i32 2)
+  ret <2 x float> %r4
+}
+
+define <2 x float> @add_f32x2_f16x2_lanes_commuted(<2 x half> %a, <2 x float> %b) {
+; CHECK-LABEL: add_f32x2_f16x2_lanes_commuted(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_lanes_commuted_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_lanes_commuted_param_1];
+; CHECK-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %b, <2 x float> %e, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_f16x2_extra_use(<2 x half> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_extra_use(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<4>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_f16x2_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-NOFTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd4, %r3, %rd2;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_extra_use(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<4>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_extra_use_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_f16x2_extra_use_param_2];
+; CHECK-FTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-FTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
+; CHECK-FTZ-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd4, %r3, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  store <2 x float> %e, ptr %p
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+; packed
+
+define <2 x float> @add_f32x2_f16x2_packed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_packed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_packed_param_0];
+; CHECK-NOFTZ-NEXT:    mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_packed_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_packed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_packed_param_0];
+; CHECK-FTZ-NEXT:    mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_packed_param_1];
+; CHECK-FTZ-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-FTZ-NEXT:    ret;
+  %x = fmul <2 x half> %a, %a
+  %e = fpext <2 x half> %x to <2 x float>
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_f16x2_packed_commuted(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_packed_commuted(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_packed_commuted_param_0];
+; CHECK-NOFTZ-NEXT:    mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_packed_commuted_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_packed_commuted(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_packed_commuted_param_0];
+; CHECK-FTZ-NEXT:    mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_packed_commuted_param_1];
+; CHECK-FTZ-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-FTZ-NEXT:    ret;
+  %x = fmul <2 x half> %a, %a
+  %e = fpext <2 x half> %x to <2 x float>
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %b, <2 x float> %e, i32 1)
+  ret <2 x float> %r
+}
+
+; generic fadd
+
+define <2 x float> @add_f32x2_f16x2_generic_lanes(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_lanes(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<2>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lanes_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_lanes_param_1];
+; CHECK-NOFTZ-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_lanes(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lanes_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_lanes_param_1];
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %r = fadd <2 x float> %e, %b
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_f16x2_generic_lanes_commuted(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_lanes_commuted(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<2>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lanes_commuted_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_lanes_commuted_param_1];
+; CHECK-NOFTZ-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_lanes_commuted(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lanes_commuted_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_lanes_commuted_param_1];
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %r = fadd <2 x float> %b, %e
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_f16x2_generic_packed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_packed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_generic_packed_param_0];
+; CHECK-NOFTZ-NEXT:    mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_packed_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_packed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_generic_packed_param_0];
+; CHECK-FTZ-NEXT:    mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_packed_param_1];
+; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r4, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %x = fmul <2 x half> %a, %a
+  %e = fpext <2 x half> %x to <2 x float>
+  %r = fadd <2 x float> %e, %b
+  ret <2 x float> %r
+}
+
+;
+;  BF16x2
+;
+
+; lanes
+
+define <2 x float> @add_f32x2_bf16x2_lanes(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-LABEL: add_f32x2_bf16x2_lanes(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_lanes_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_lanes_param_1];
+; CHECK-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NEXT:    add.rz.f32x2.bf16x2.f32x2 %rd3, %r1, %rd2;
+; CHECK-NEXT:    add.rm.f32x2.bf16x2.f32x2 %rd4, %r1, %rd3;
+; CHECK-NEXT:    add.rp.f32x2.bf16x2.f32x2 %rd5, %r1, %rd4;
+; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %r1, i32 0)
+  %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %r2, i32 3)
+  %r4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %r3, i32 2)
+  ret <2 x float> %r4
+}
+
+define <2 x float> @add_f32x2_bf16x2_lanes_commuted(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-LABEL: add_f32x2_bf16x2_lanes_commuted(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_lanes_commuted_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_lanes_commuted_param_1];
+; CHECK-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %b, <2 x float> %e, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_extra_use(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<4>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_bf16x2_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-NOFTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd4, %r3, %rd2;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_extra_use(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<4>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_extra_use_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_bf16x2_extra_use_param_2];
+; CHECK-FTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-FTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
+; CHECK-FTZ-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd4, %r3, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  store <2 x float> %e, ptr %p
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+; packed
+
+define <2 x float> @add_f32x2_bf16x2_packed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-LABEL: add_f32x2_bf16x2_packed(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b32 %r1, [add_f32x2_bf16x2_packed_param_0];
+; CHECK-NEXT:    mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_packed_param_1];
+; CHECK-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NEXT:    ret;
+  %x = fmul <2 x bfloat> %a, %a
+  %e = fpext <2 x bfloat> %x to <2 x float>
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2_packed_commuted(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-LABEL: add_f32x2_bf16x2_packed_commuted(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b32 %r1, [add_f32x2_bf16x2_packed_commuted_param_0];
+; CHECK-NEXT:    mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_packed_commuted_param_1];
+; CHECK-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NEXT:    ret;
+  %x = fmul <2 x bfloat> %a, %a
+  %e = fpext <2 x bfloat> %x to <2 x float>
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %b, <2 x float> %e, i32 1)
+  ret <2 x float> %r
+}
+
+; generic fadd
+
+define <2 x float> @add_f32x2_bf16x2_generic_lanes(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_lanes(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<2>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lanes_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_lanes_param_1];
+; CHECK-NOFTZ-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_lanes(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lanes_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_lanes_param_1];
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %r = fadd <2 x float> %e, %b
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2_generic_lanes_commuted(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_lanes_commuted(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<2>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lanes_commuted_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_lanes_commuted_param_1];
+; CHECK-NOFTZ-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_lanes_commuted(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lanes_commuted_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_lanes_commuted_param_1];
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %r = fadd <2 x float> %b, %e
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2_generic_packed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_packed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_bf16x2_generic_packed_param_0];
+; CHECK-NOFTZ-NEXT:    mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_packed_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_packed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_bf16x2_generic_packed_param_0];
+; CHECK-FTZ-NEXT:    mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_packed_param_1];
+; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r3, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r4, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %x = fmul <2 x bfloat> %a, %a
+  %e = fpext <2 x bfloat> %x to <2 x float>
+  %r = fadd <2 x float> %e, %b
+  ret <2 x float> %r
+}

>From 2007ca064a9a849dafd404f00f3a67d45e138fe6 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 8 Sep 2026 04:27:05 +0000
Subject: [PATCH 2/7] change ordering and add comment

---
 llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 38 +++++++++++++-----------
 1 file changed, 21 insertions(+), 17 deletions(-)

diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index c047c4fd4f94b..a723af51740ac 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2541,23 +2541,6 @@ foreach rnd = FPRoundingModes in {
   }
 }
 
-// mixed precision, downconverting
-foreach t = [F16X2RT, BF16X2RT] in {
-  defvar ftz = !if(!eq(t, F16X2RT), "ftz", "");
-  defvar AddOp = !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", ftz]>.ret);
-  defvar CvtOp = !cast<Intrinsic>("int_nvvm_ff2" # t.PtxType # "_rz");
-
-  def INT_NVVM_MIXED_ADD_rz_ # t.PtxType # _f32x2 :
-    BasicNVPTXInst<(outs t.RC:$dst), (ins B64:$a, B64:$b),
-      StrJoin<".", ["add.rz", ftz, t.PtxType, "f32x2", "f32x2"]>.ret,
-      // The cvt packs its first argument into the high half of the result.
-      [(set t.Ty:$dst,
-        (CvtOp (extractelt (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 1),
-               (extractelt (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 0),
-               /*pzo=*/0))]>,
-    Requires<[SM107f]>;
-}
-
 // Pattern for fadd when there is no FTZ flag
 let Predicates = [SM100, doNoF32FTZ] in {
   def : Pat<(f32 (fadd (f32 (fpextend f16:$a)), f32:$b)),
@@ -2579,6 +2562,27 @@ let Predicates = [SM107f, doNoF32FTZ] in
               (Inst (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
   }
 
+
+// mixed precision - downconverting
+
+// f16x2 - rz rounding mode, only with ftz
+// bf16x2 - rz rounding mode, without ftz
+foreach t = [F16X2RT, BF16X2RT] in {
+  defvar ftz = !if(!eq(t, F16X2RT), "ftz", "");
+  defvar AddOp = !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", ftz]>.ret);
+  defvar CvtOp = !cast<Intrinsic>("int_nvvm_ff2" # t.PtxType # "_rz");
+
+  def INT_NVVM_MIXED_ADD_rz_ # t.PtxType # _f32x2 :
+    BasicNVPTXInst<(outs t.RC:$dst), (ins B64:$a, B64:$b),
+      StrJoin<".", ["add.rz", ftz, t.PtxType, "f32x2", "f32x2"]>.ret,
+      // The cvt packs its first argument into the high half of the result.
+      [(set t.Ty:$dst,
+        (CvtOp (extractelt (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 1),
+               (extractelt (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 0),
+               /*pzo=*/0))]>,
+    Requires<[SM107f]>;
+}
+
 //
 // Sub
 //

>From 57e2cdf3743b7a3475380820c749f7732c68f28d Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 8 Sep 2026 06:14:49 +0000
Subject: [PATCH 3/7] add sub patterns

---
 llvm/lib/Target/NVPTX/NVPTXIntrinsics.td      |  54 ++
 .../mixed-precision-sub-f32x2-downconvert.ll  | 346 ++++++++++++
 .../mixed-precision-sub-f32x2-upconvert.ll    | 524 ++++++++++++++++++
 3 files changed, 924 insertions(+)
 create mode 100644 llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert.ll

diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index a723af51740ac..a653c70050e8b 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2644,6 +2644,7 @@ def INT_NVVM_SUB_D :
     [(set f64:$dst, (int_nvvm_fadd f64:$a, (f64 (fneg f64:$b)), timm:$rnd))]>;
 
 // mixed precision
+
 foreach rnd = FPRoundingModes in {
   defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
 
@@ -2657,6 +2658,26 @@ foreach rnd = FPRoundingModes in {
              (f32 (fpextend type:$a)),
              (f32 (fneg f32:$b)), rnd_imm))]>,
         Requires<[SM100]>;
+
+  foreach t = [F16X2RT, BF16X2RT] in {
+    // combineFAddWithNeg has already folded the fneg into a sub node.
+    defvar SubOp = !cast<SDNode>("sub_" # rnd);
+
+    def INT_NVVM_MIXED_SUB_ # rnd # _f32x2_ # t.PtxType :
+      BasicNVPTXInst<(outs B64:$dst), (ins t.RC:$a, B64:$b),
+        StrJoin<".", ["sub", rnd, "f32x2", t.PtxType, "f32x2"]>.ret,
+        [(set v2f32:$dst,
+          (SubOp (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b))]>,
+      Requires<[SM107f]>;
+
+    def : Pat<(v2f32 (SubOp
+                       (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
+                                                  t.Ty.ElementType:$a1),
+                       v2f32:$b)),
+              (!cast<Instruction>("INT_NVVM_MIXED_SUB_" # rnd # "_f32x2_" #
+                                  t.PtxType) (t.Ty (V2I16toI32 $a0, $a1)), $b)>,
+      Requires<[SM107f]>;
+  }
 }
 
 // Pattern for fsub when there is no FTZ flag
@@ -2667,6 +2688,39 @@ let Predicates = [SM100, doNoF32FTZ] in {
             (INT_NVVM_MIXED_SUB_rn_f32_bf16 B16:$a, B32:$b)>;
 }
 
+let Predicates = [SM107f, doNoF32FTZ] in
+  foreach t = [F16X2RT, BF16X2RT] in {
+    defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_SUB_rn_f32x2_" # t.PtxType);
+
+    def : Pat<(v2f32 (fsub (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b)),
+              (Inst $a, $b)>;
+
+    def : Pat<(v2f32 (fsub (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
+                                                      t.Ty.ElementType:$a1),
+                           v2f32:$b)),
+              (Inst (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
+  }
+
+// mixed precision - downconverting
+
+// f16x2 - rz rounding mode, only with ftz
+// bf16x2 - rz rounding mode, without ftz
+foreach t = [F16X2RT, BF16X2RT] in {
+  defvar ftz = !if(!eq(t, F16X2RT), "ftz", "");
+  defvar SubOp = !cast<SDNode>(StrJoin<"_", ["sub", "rz", ftz]>.ret);
+  defvar CvtOp = !cast<Intrinsic>("int_nvvm_ff2" # t.PtxType # "_rz");
+
+  def INT_NVVM_MIXED_SUB_rz_ # t.PtxType # _f32x2 :
+    BasicNVPTXInst<(outs t.RC:$dst), (ins B64:$a, B64:$b),
+      StrJoin<".", ["sub.rz", ftz, t.PtxType, "f32x2", "f32x2"]>.ret,
+      // The cvt packs its first argument into the high half of the result.
+      [(set t.Ty:$dst,
+        (CvtOp (extractelt (SubOp v2f32:$a, v2f32:$b), 1),
+               (extractelt (SubOp v2f32:$a, v2f32:$b), 0),
+               /*pzo=*/0))]>,
+    Requires<[SM107f]>;
+}
+
 //
 // BFIND
 //
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert.ll
new file mode 100644
index 0000000000000..ae71d63dc5da8
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert.ll
@@ -0,0 +1,346 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+;
+;  F16x2
+;
+
+define <2 x half> @sub_rz_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_param_1];
+; CHECK-NEXT:    sub.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_negated_minuend(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_negated_minuend(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_negated_minuend_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_negated_minuend_param_1];
+; CHECK-NEXT:    sub.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd2, %rd1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %na = fneg <2 x float> %a
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %na, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_extra_use(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_extra_use_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_extra_use_param_1];
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    ld.param::func.b64 %rd4, [sub_rz_f16x2_f32x2_extra_use_param_2];
+; CHECK-NEXT:    st.b64 [%rd4], %rd3;
+; CHECK-NEXT:    sub.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  store <2 x float> %sum, ptr %p
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+; invalid patterns
+
+define <2 x half> @sub_rz_f16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_swapped_halves(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_swapped_halves_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_swapped_halves_param_1];
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %lo, float %hi, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_no_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_no_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_no_ftz_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_no_ftz_param_1];
+; CHECK-NEXT:    sub.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @sub_rn_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rn_f16x2_f32x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rn_f16x2_f32x2_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rn_f16x2_f32x2_param_1];
+; CHECK-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 1)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_rn_convert(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_rn_convert(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_rn_convert_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_rn_convert_param_1];
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rn.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_relu(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_relu(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_relu_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_relu_param_1];
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.relu.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_distinct_subs(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_distinct_subs(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_distinct_subs_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_distinct_subs_param_1];
+; CHECK-NEXT:    ld.param::func.b64 %rd3, [sub_rz_f16x2_f32x2_distinct_subs_param_2];
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd5, %rd1, %rd3;
+; CHECK-NEXT:    mov.b64 {%r1, _}, %rd4;
+; CHECK-NEXT:    mov.b64 {_, %r2}, %rd5;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %nc = fneg <2 x float> %c
+  %sum1 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %sum2 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nc, i32 0)
+  %lo = extractelement <2 x float> %sum1, i32 0
+  %hi = extractelement <2 x float> %sum2, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+;
+;  BF16x2
+;
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_param_1];
+; CHECK-NEXT:    sub.rz.bf16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2_negated_minuend(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2_negated_minuend(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_negated_minuend_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_negated_minuend_param_1];
+; CHECK-NEXT:    sub.rz.bf16x2.f32x2.f32x2 %r1, %rd2, %rd1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %na = fneg <2 x float> %a
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %na, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2_extra_use(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_extra_use_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_extra_use_param_1];
+; CHECK-NEXT:    sub.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    ld.param::func.b64 %rd4, [sub_rz_bf16x2_f32x2_extra_use_param_2];
+; CHECK-NEXT:    st.b64 [%rd4], %rd3;
+; CHECK-NEXT:    sub.rz.bf16x2.f32x2.f32x2 %r1, %rd1, %rd2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  store <2 x float> %sum, ptr %p
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
+
+; invalid patterns
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2_swapped_halves(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_swapped_halves_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_swapped_halves_param_1];
+; CHECK-NEXT:    sub.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %lo, float %hi, i1 false)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_ftz_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_ftz_param_1];
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @sub_rn_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rn_bf16x2_f32x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rn_bf16x2_f32x2_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rn_bf16x2_f32x2_param_1];
+; CHECK-NEXT:    sub.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 1)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert.ll
new file mode 100644
index 0000000000000..9872325d3a302
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert.ll
@@ -0,0 +1,524 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck --check-prefixes=CHECK,CHECK-NOFTZ %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | FileCheck --check-prefixes=CHECK,CHECK-FTZ %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | %ptxas-verify -arch=sm_107f %}
+
+;
+;  F16x2
+;
+
+; lanes
+
+define <2 x float> @sub_f32x2_f16x2_lanes(<2 x half> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_f32x2_f16x2_lanes(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_lanes_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_lanes_param_1];
+; CHECK-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT:    sub.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NEXT:    sub.rz.f32x2.f16x2.f32x2 %rd3, %r1, %rd2;
+; CHECK-NEXT:    sub.rm.f32x2.f16x2.f32x2 %rd4, %r1, %rd3;
+; CHECK-NEXT:    sub.rp.f32x2.f16x2.f32x2 %rd5, %r1, %rd4;
+; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %nb = fneg <2 x float> %b
+  %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+  %n1 = fneg <2 x float> %r1
+  %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %n1, i32 0)
+  %n2 = fneg <2 x float> %r2
+  %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %n2, i32 3)
+  %n3 = fneg <2 x float> %r3
+  %r4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %n3, i32 2)
+  ret <2 x float> %r4
+}
+
+define <2 x float> @sub_f32x2_f16x2_extra_use(<2 x half> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_extra_use(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<4>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_f16x2_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-NOFTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2.f16x2.f32x2 %rd4, %r3, %rd2;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_extra_use(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<4>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_extra_use_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_f16x2_extra_use_param_2];
+; CHECK-FTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-FTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
+; CHECK-FTZ-NEXT:    sub.rn.f32x2.f16x2.f32x2 %rd4, %r3, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  store <2 x float> %e, ptr %p
+  %nb = fneg <2 x float> %b
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+  ret <2 x float> %r
+}
+
+; packed
+
+define <2 x float> @sub_f32x2_f16x2_packed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_packed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [sub_f32x2_f16x2_packed_param_0];
+; CHECK-NOFTZ-NEXT:    mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_packed_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_packed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [sub_f32x2_f16x2_packed_param_0];
+; CHECK-FTZ-NEXT:    mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_packed_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-FTZ-NEXT:    ret;
+  %x = fmul <2 x half> %a, %a
+  %e = fpext <2 x half> %x to <2 x float>
+  %nb = fneg <2 x float> %b
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+  ret <2 x float> %r
+}
+
+; generic fsub
+
+define <2 x float> @sub_f32x2_f16x2_generic_lanes(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_lanes(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<2>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_lanes_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_lanes_param_1];
+; CHECK-NOFTZ-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_lanes(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_lanes_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_lanes_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %r = fsub <2 x float> %e, %b
+  ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_f16x2_generic_packed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_packed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [sub_f32x2_f16x2_generic_packed_param_0];
+; CHECK-NOFTZ-NEXT:    mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_packed_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2.f16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_packed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [sub_f32x2_f16x2_generic_packed_param_0];
+; CHECK-FTZ-NEXT:    mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_packed_param_1];
+; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r4, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %x = fmul <2 x half> %a, %a
+  %e = fpext <2 x half> %x to <2 x float>
+  %r = fsub <2 x float> %e, %b
+  ret <2 x float> %r
+}
+
+; invalid patterns
+
+define <2 x float> @sub_f32x2_f16x2_reversed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_reversed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_reversed_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_reversed_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_reversed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_reversed_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_reversed_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %ne = fneg <2 x float> %e
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %ne, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_f16x2_generic_reversed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_reversed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_reversed_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_reversed_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_reversed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_reversed_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_reversed_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %r = fsub <2 x float> %b, %e
+  ret <2 x float> %r
+}
+
+;
+;  BF16x2
+;
+
+; lanes
+
+define <2 x float> @sub_f32x2_bf16x2_lanes(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_f32x2_bf16x2_lanes(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_lanes_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_lanes_param_1];
+; CHECK-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT:    sub.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NEXT:    sub.rz.f32x2.bf16x2.f32x2 %rd3, %r1, %rd2;
+; CHECK-NEXT:    sub.rm.f32x2.bf16x2.f32x2 %rd4, %r1, %rd3;
+; CHECK-NEXT:    sub.rp.f32x2.bf16x2.f32x2 %rd5, %r1, %rd4;
+; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %nb = fneg <2 x float> %b
+  %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+  %n1 = fneg <2 x float> %r1
+  %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %n1, i32 0)
+  %n2 = fneg <2 x float> %r2
+  %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %n2, i32 3)
+  %n3 = fneg <2 x float> %r3
+  %r4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %n3, i32 2)
+  ret <2 x float> %r4
+}
+
+define <2 x float> @sub_f32x2_bf16x2_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_extra_use(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<4>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_bf16x2_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-NOFTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2.bf16x2.f32x2 %rd4, %r3, %rd2;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_extra_use(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<4>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_extra_use_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_bf16x2_extra_use_param_2];
+; CHECK-FTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-FTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
+; CHECK-FTZ-NEXT:    sub.rn.f32x2.bf16x2.f32x2 %rd4, %r3, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  store <2 x float> %e, ptr %p
+  %nb = fneg <2 x float> %b
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+  ret <2 x float> %r
+}
+
+; packed
+
+define <2 x float> @sub_f32x2_bf16x2_packed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_f32x2_bf16x2_packed(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<3>;
+; CHECK-NEXT:    .reg .b64 %rd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b32 %r1, [sub_f32x2_bf16x2_packed_param_0];
+; CHECK-NEXT:    mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_packed_param_1];
+; CHECK-NEXT:    sub.rn.f32x2.bf16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NEXT:    ret;
+  %x = fmul <2 x bfloat> %a, %a
+  %e = fpext <2 x bfloat> %x to <2 x float>
+  %nb = fneg <2 x float> %b
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+  ret <2 x float> %r
+}
+
+; generic fsub
+
+define <2 x float> @sub_f32x2_bf16x2_generic_lanes(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_lanes(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<2>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_lanes_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_lanes_param_1];
+; CHECK-NOFTZ-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_lanes(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_lanes_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_lanes_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %r = fsub <2 x float> %e, %b
+  ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_bf16x2_generic_packed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_packed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<3>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [sub_f32x2_bf16x2_generic_packed_param_0];
+; CHECK-NOFTZ-NEXT:    mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_packed_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2.bf16x2.f32x2 %rd2, %r2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd2;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_packed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [sub_f32x2_bf16x2_generic_packed_param_0];
+; CHECK-FTZ-NEXT:    mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_packed_param_1];
+; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r3, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r4, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %x = fmul <2 x bfloat> %a, %a
+  %e = fpext <2 x bfloat> %x to <2 x float>
+  %r = fsub <2 x float> %e, %b
+  ret <2 x float> %r
+}
+
+; invalid patterns
+
+define <2 x float> @sub_f32x2_bf16x2_reversed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_reversed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_reversed_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_reversed_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_reversed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_reversed_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_reversed_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %ne = fneg <2 x float> %e
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %ne, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_bf16x2_generic_reversed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_reversed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_reversed_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_reversed_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_reversed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_reversed_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_reversed_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %r = fsub <2 x float> %b, %e
+  ret <2 x float> %r
+}

>From 035c4cba53cabe4be5c7d4918bb8f2f11d4c473c Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Tue, 8 Sep 2026 09:29:42 +0000
Subject: [PATCH 4/7] add invalid pattern tests for upconvert add

---
 .../mixed-precision-add-f32x2-upconvert.ll    | 158 ++++++++++++++++++
 1 file changed, 158 insertions(+)

diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll
index 35d53eebf8ef0..27708d4177786 100644
--- a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll
@@ -273,6 +273,85 @@ define <2 x float> @add_f32x2_f16x2_generic_packed(<2 x half> %a, <2 x float> %b
   ret <2 x float> %r
 }
 
+; invalid patterns
+
+define <2 x float> @add_f32x2_f16x2_ftz(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_ftz(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_ftz_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_ftz_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_ftz(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_ftz_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_ftz_param_1];
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_f16x2_partial(<2 x half> %a, float %c, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_partial(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_partial_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b16 %rs1, [add_f32x2_f16x2_partial_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_partial_param_2];
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd2, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_partial(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<2>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_partial_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b16 %rs1, [add_f32x2_f16x2_partial_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_partial_param_2];
+; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r2, %r1};
+; CHECK-FTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %a0 = extractelement <2 x half> %a, i32 0
+  %e0 = fpext half %a0 to float
+  %v0 = insertelement <2 x float> poison, float %e0, i32 0
+  %v1 = insertelement <2 x float> %v0, float %c, i32 1
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %v1, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
 ;
 ;  BF16x2
 ;
@@ -515,3 +594,82 @@ define <2 x float> @add_f32x2_bf16x2_generic_packed(<2 x bfloat> %a, <2 x float>
   %r = fadd <2 x float> %e, %b
   ret <2 x float> %r
 }
+
+; invalid patterns
+
+define <2 x float> @add_f32x2_bf16x2_ftz(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_ftz(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_ftz_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_ftz_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_ftz(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_ftz_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_ftz_param_1];
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2_partial(<2 x bfloat> %a, float %c, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_partial(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_bf16x2_partial_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b16 %rs1, [add_f32x2_bf16x2_partial_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_partial_param_2];
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd2, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_partial(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<2>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_bf16x2_partial_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b16 %rs1, [add_f32x2_bf16x2_partial_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_partial_param_2];
+; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r2, %r1};
+; CHECK-FTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %a0 = extractelement <2 x bfloat> %a, i32 0
+  %e0 = fpext bfloat %a0 to float
+  %v0 = insertelement <2 x float> poison, float %e0, i32 0
+  %v1 = insertelement <2 x float> %v0, float %c, i32 1
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %v1, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}

>From a843d961edb90ac7adba8c9de6a0735129b5033f Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 9 Sep 2026 06:47:27 +0000
Subject: [PATCH 5/7] address comments

---
 llvm/lib/Target/NVPTX/NVPTXInstrInfo.td  |  1 +
 llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 96 ++++++++++++------------
 2 files changed, 49 insertions(+), 48 deletions(-)

diff --git a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
index 846bcc0557464..bd2e8dd532d87 100644
--- a/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
+++ b/llvm/lib/Target/NVPTX/NVPTXInstrInfo.td
@@ -181,6 +181,7 @@ def hasTensormapReplaceSupport : SubtargetPredicate;
 //  - TMA S2G im2col_w mode support
 //  - tcgen05.commit shared mem A variants.
 //  - conversions involving ue5m3x2
+//  - f16x2/bf16x2 mixed precision support
 def hasRubinFamilySupport : PredOr<[SM107f]>;
 
 // Checks tcgen05.shift instruction support.
diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index a653c70050e8b..137ed64cd10b4 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2509,7 +2509,7 @@ def fpextend_v2f32_from_lanes : PatFrag<(ops node:$a0, node:$a1),
                                           (f32 (fpextend node:$a1))))>;
 
 foreach rnd = FPRoundingModes in {
-  defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
+  defvar rnd_imm = !cast<TImmLeaf>("fp_rnd_" # rnd # "_imm");
 
   foreach sat = ["", "sat"] in
     foreach type = [f16, bf16] in
@@ -2522,22 +2522,22 @@ foreach rnd = FPRoundingModes in {
              f32:$b, rnd_imm))]>,
         Requires<[SM100]>;
 
-  foreach t = [F16X2RT, BF16X2RT] in {
-    def INT_NVVM_MIXED_ADD_ # rnd # _f32x2_ # t.PtxType :
-      BasicNVPTXInst<(outs B64:$dst), (ins t.RC:$a, B64:$b),
-        StrJoin<".", ["add", rnd, "f32x2", t.PtxType, "f32x2"]>.ret,
-        [(set v2f32:$dst,
-          (int_nvvm_fadd (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b,
-                         rnd_imm))]>,
-      Requires<[SM107f]>;
-
-    def : Pat<(v2f32 (int_nvvm_fadd
-                       (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
-                                                  t.Ty.ElementType:$a1),
-                       v2f32:$b, rnd_imm)),
-              (!cast<Instruction>("INT_NVVM_MIXED_ADD_" # rnd # "_f32x2_" #
-                                  t.PtxType) (t.Ty (V2I16toI32 $a0, $a1)), $b)>,
-      Requires<[SM107f]>;
+  let Predicates = [hasRubinFamilySupport] in {
+    foreach t = [F16X2RT, BF16X2RT] in {
+      def INT_NVVM_MIXED_ADD_ # rnd # _f32x2_ # t.PtxType :
+        BasicNVPTXInst<(outs B64:$dst), (ins B32:$a, B64:$b),
+          "add." # rnd # ".f32x2." # t.PtxType # ".f32x2",
+          [(set v2f32:$dst,
+            (int_nvvm_fadd (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b,
+                          rnd_imm))]>;
+
+      def : Pat<(v2f32 (int_nvvm_fadd
+                        (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
+                                                    t.Ty.ElementType:$a1),
+                        v2f32:$b, rnd_imm)),
+                (!cast<Instruction>("INT_NVVM_MIXED_ADD_" # rnd # "_f32x2_" #
+                                    t.PtxType) (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
+    }
   }
 }
 
@@ -2549,7 +2549,7 @@ let Predicates = [SM100, doNoF32FTZ] in {
             (INT_NVVM_MIXED_ADD_rn_f32_bf16 B16:$a, B32:$b)>;
 }
 
-let Predicates = [SM107f, doNoF32FTZ] in
+let Predicates = [hasRubinFamilySupport, doNoF32FTZ] in
   foreach t = [F16X2RT, BF16X2RT] in {
     defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_ADD_rn_f32x2_" # t.PtxType);
 
@@ -2568,19 +2568,19 @@ let Predicates = [SM107f, doNoF32FTZ] in
 // f16x2 - rz rounding mode, only with ftz
 // bf16x2 - rz rounding mode, without ftz
 foreach t = [F16X2RT, BF16X2RT] in {
-  defvar ftz = !if(!eq(t, F16X2RT), "ftz", "");
-  defvar AddOp = !cast<Intrinsic>(StrJoin<"_", ["int_nvvm_fadd", ftz]>.ret);
+  defvar ftz = !if(!eq(t, F16X2RT), ".ftz", "");
+  defvar AddOp = !cast<Intrinsic>("int_nvvm_fadd" # !if(!eq(ftz, ""), "", "_ftz"));
   defvar CvtOp = !cast<Intrinsic>("int_nvvm_ff2" # t.PtxType # "_rz");
 
   def INT_NVVM_MIXED_ADD_rz_ # t.PtxType # _f32x2 :
-    BasicNVPTXInst<(outs t.RC:$dst), (ins B64:$a, B64:$b),
-      StrJoin<".", ["add.rz", ftz, t.PtxType, "f32x2", "f32x2"]>.ret,
+    BasicNVPTXInst<(outs B32:$dst), (ins B64:$a, B64:$b),
+      "add.rz" # ftz # "." # t.PtxType # ".f32x2.f32x2",
       // The cvt packs its first argument into the high half of the result.
       [(set t.Ty:$dst,
         (CvtOp (extractelt (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 1),
                (extractelt (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 0),
                /*pzo=*/0))]>,
-    Requires<[SM107f]>;
+    Requires<[hasRubinFamilySupport]>;
 }
 
 //
@@ -2646,7 +2646,7 @@ def INT_NVVM_SUB_D :
 // mixed precision
 
 foreach rnd = FPRoundingModes in {
-  defvar rnd_imm = !cast<TImmLeaf>(StrJoin<"_", ["fp_rnd", rnd, "imm"]>.ret);
+  defvar rnd_imm = !cast<TImmLeaf>("fp_rnd_" # rnd # "_imm");
 
   foreach sat = ["", "sat"] in
     foreach type = [f16, bf16] in
@@ -2659,24 +2659,24 @@ foreach rnd = FPRoundingModes in {
              (f32 (fneg f32:$b)), rnd_imm))]>,
         Requires<[SM100]>;
 
-  foreach t = [F16X2RT, BF16X2RT] in {
-    // combineFAddWithNeg has already folded the fneg into a sub node.
-    defvar SubOp = !cast<SDNode>("sub_" # rnd);
-
-    def INT_NVVM_MIXED_SUB_ # rnd # _f32x2_ # t.PtxType :
-      BasicNVPTXInst<(outs B64:$dst), (ins t.RC:$a, B64:$b),
-        StrJoin<".", ["sub", rnd, "f32x2", t.PtxType, "f32x2"]>.ret,
-        [(set v2f32:$dst,
-          (SubOp (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b))]>,
-      Requires<[SM107f]>;
-
-    def : Pat<(v2f32 (SubOp
-                       (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
-                                                  t.Ty.ElementType:$a1),
-                       v2f32:$b)),
-              (!cast<Instruction>("INT_NVVM_MIXED_SUB_" # rnd # "_f32x2_" #
-                                  t.PtxType) (t.Ty (V2I16toI32 $a0, $a1)), $b)>,
-      Requires<[SM107f]>;
+  let Predicates = [hasRubinFamilySupport] in {
+    foreach t = [F16X2RT, BF16X2RT] in {
+      // combineFAddWithNeg has already folded the fneg into a sub node.
+      defvar SubOp = !cast<SDNode>("sub_" # rnd);
+
+      def INT_NVVM_MIXED_SUB_ # rnd # _f32x2_ # t.PtxType :
+        BasicNVPTXInst<(outs B64:$dst), (ins B32:$a, B64:$b),
+          "sub." # rnd # ".f32x2." # t.PtxType # ".f32x2",
+          [(set v2f32:$dst,
+            (SubOp (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b))]>;
+
+      def : Pat<(v2f32 (SubOp
+                        (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
+                                                    t.Ty.ElementType:$a1),
+                        v2f32:$b)),
+                (!cast<Instruction>("INT_NVVM_MIXED_SUB_" # rnd # "_f32x2_" #
+                                    t.PtxType) (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
+    }
   }
 }
 
@@ -2688,7 +2688,7 @@ let Predicates = [SM100, doNoF32FTZ] in {
             (INT_NVVM_MIXED_SUB_rn_f32_bf16 B16:$a, B32:$b)>;
 }
 
-let Predicates = [SM107f, doNoF32FTZ] in
+let Predicates = [hasRubinFamilySupport, doNoF32FTZ] in
   foreach t = [F16X2RT, BF16X2RT] in {
     defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_SUB_rn_f32x2_" # t.PtxType);
 
@@ -2706,19 +2706,19 @@ let Predicates = [SM107f, doNoF32FTZ] in
 // f16x2 - rz rounding mode, only with ftz
 // bf16x2 - rz rounding mode, without ftz
 foreach t = [F16X2RT, BF16X2RT] in {
-  defvar ftz = !if(!eq(t, F16X2RT), "ftz", "");
-  defvar SubOp = !cast<SDNode>(StrJoin<"_", ["sub", "rz", ftz]>.ret);
+  defvar ftz = !if(!eq(t, F16X2RT), ".ftz", "");
+  defvar SubOp = !cast<SDNode>("sub_rz" # !if(!eq(ftz, ""), "", "_ftz"));
   defvar CvtOp = !cast<Intrinsic>("int_nvvm_ff2" # t.PtxType # "_rz");
 
   def INT_NVVM_MIXED_SUB_rz_ # t.PtxType # _f32x2 :
-    BasicNVPTXInst<(outs t.RC:$dst), (ins B64:$a, B64:$b),
-      StrJoin<".", ["sub.rz", ftz, t.PtxType, "f32x2", "f32x2"]>.ret,
+    BasicNVPTXInst<(outs B32:$dst), (ins B64:$a, B64:$b),
+      "sub.rz" # ftz # "." # t.PtxType # ".f32x2.f32x2",
       // The cvt packs its first argument into the high half of the result.
       [(set t.Ty:$dst,
         (CvtOp (extractelt (SubOp v2f32:$a, v2f32:$b), 1),
                (extractelt (SubOp v2f32:$a, v2f32:$b), 0),
                /*pzo=*/0))]>,
-    Requires<[SM107f]>;
+    Requires<[hasRubinFamilySupport]>;
 }
 
 //

>From 5b0cc6000006bda4d279c1298a9374b10404fd73 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Wed, 9 Sep 2026 10:58:43 +0000
Subject: [PATCH 6/7] constrain folding only to single-use

---
 llvm/lib/Target/NVPTX/NVPTXIntrinsics.td      |  44 +-
 ...precision-add-f32x2-downconvert-invalid.ll | 264 +++++++
 .../mixed-precision-add-f32x2-downconvert.ll  | 243 ------
 ...d-precision-add-f32x2-upconvert-invalid.ll | 711 ++++++++++++++++++
 .../mixed-precision-add-f32x2-upconvert.ll    | 324 +++-----
 ...precision-sub-f32x2-downconvert-invalid.ll | 276 +++++++
 .../mixed-precision-sub-f32x2-downconvert.ll  | 255 -------
 ...d-precision-sub-f32x2-upconvert-invalid.ll | 535 +++++++++++++
 .../mixed-precision-sub-f32x2-upconvert.ll    | 350 +++------
 9 files changed, 2023 insertions(+), 979 deletions(-)
 create mode 100644 llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert-invalid.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert-invalid.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert-invalid.ll
 create mode 100644 llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert-invalid.ll

diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 137ed64cd10b4..424f4a80be890 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2496,17 +2496,33 @@ def INT_NVVM_ADD_D :
 
 // mixed precision
 
+// match only when the fold removes at least one explicit conversion, and the
+// extended vector is not shared.
+class FoldableFPExtendV2F32<dag ops, dag frag>
+  : PatFrag<ops, frag, [{
+      return N->hasOneUse() && (N->getOperand(0)->hasOneUse() ||
+                                N->getOperand(1)->hasOneUse());
+    }]>;
+
 // FP_EXTEND to v2f32 is scalarized before isel, leaving one of two shapes
 // behind depending on whether the source lanes share a register.
-def fpextend_v2f32_from_packed : PatFrag<(ops node:$a),
-                                         (v2f32 (build_vector
-                                           (f32 (fpextend (extractelt node:$a, 0))),
-                                           (f32 (fpextend (extractelt node:$a, 1)))))>;
-
-def fpextend_v2f32_from_lanes : PatFrag<(ops node:$a0, node:$a1),
-                                        (v2f32 (build_vector
-                                          (f32 (fpextend node:$a0)),
-                                          (f32 (fpextend node:$a1))))>;
+def fpextend_v2f32_from_packed
+  : FoldableFPExtendV2F32<(ops node:$a),
+                          (v2f32 (build_vector
+                            (f32 (fpextend (extractelt node:$a, 0))),
+                            (f32 (fpextend (extractelt node:$a, 1)))))>;
+
+def fpextend_v2f32_from_lanes
+  : FoldableFPExtendV2F32<(ops node:$a0, node:$a1),
+                          (v2f32 (build_vector
+                            (f32 (fpextend node:$a0)),
+                            (f32 (fpextend node:$a1))))>;
+
+// fuse downconverting add/sub only when the result of the add/sub is solely
+// used for downconverting.
+def extractelt_only_for_cvt
+  : PatFrag<(ops node:$vec, node:$lane), (extractelt node:$vec, node:$lane),
+            [{ return N->hasOneUse() && N->getOperand(0)->hasNUsesOfValue(2, 0); }]>;
 
 foreach rnd = FPRoundingModes in {
   defvar rnd_imm = !cast<TImmLeaf>("fp_rnd_" # rnd # "_imm");
@@ -2577,8 +2593,10 @@ foreach t = [F16X2RT, BF16X2RT] in {
       "add.rz" # ftz # "." # t.PtxType # ".f32x2.f32x2",
       // The cvt packs its first argument into the high half of the result.
       [(set t.Ty:$dst,
-        (CvtOp (extractelt (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 1),
-               (extractelt (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 0),
+        (CvtOp (extractelt_only_for_cvt
+                 (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 1),
+               (extractelt_only_for_cvt
+                 (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 0),
                /*pzo=*/0))]>,
     Requires<[hasRubinFamilySupport]>;
 }
@@ -2715,8 +2733,8 @@ foreach t = [F16X2RT, BF16X2RT] in {
       "sub.rz" # ftz # "." # t.PtxType # ".f32x2.f32x2",
       // The cvt packs its first argument into the high half of the result.
       [(set t.Ty:$dst,
-        (CvtOp (extractelt (SubOp v2f32:$a, v2f32:$b), 1),
-               (extractelt (SubOp v2f32:$a, v2f32:$b), 0),
+        (CvtOp (extractelt_only_for_cvt (SubOp v2f32:$a, v2f32:$b), 1),
+               (extractelt_only_for_cvt (SubOp v2f32:$a, v2f32:$b), 0),
                /*pzo=*/0))]>,
     Requires<[hasRubinFamilySupport]>;
 }
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert-invalid.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert-invalid.ll
new file mode 100644
index 0000000000000..6d450a197a86e
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert-invalid.ll
@@ -0,0 +1,264 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+;
+; Cases where the mixed-precision instruction must not be selected.
+;
+
+;
+;  F16x2
+;
+
+; multiple uses
+
+define <2 x half> @add_rz_f16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_extra_use(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_extra_use_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_extra_use_param_1];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    ld.param::func.b64 %rd4, [add_rz_f16x2_f32x2_extra_use_param_2];
+; CHECK-NEXT:    st.b64 [%rd4], %rd3;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  store <2 x float> %sum, ptr %p
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+; unsupported forms
+
+define <2 x half> @add_rz_f16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_swapped_halves(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_swapped_halves_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_swapped_halves_param_1];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %lo, float %hi, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_no_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_no_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_no_ftz_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_no_ftz_param_1];
+; CHECK-NEXT:    add.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @add_rn_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rn_f16x2_f32x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rn_f16x2_f32x2_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rn_f16x2_f32x2_param_1];
+; CHECK-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_rn_convert(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_rn_convert(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_rn_convert_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_rn_convert_param_1];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rn.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_relu(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_relu(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_relu_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_relu_param_1];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.relu.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @add_rz_f16x2_f32x2_distinct_adds(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: add_rz_f16x2_f32x2_distinct_adds(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_distinct_adds_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_distinct_adds_param_1];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    ld.param::func.b64 %rd4, [add_rz_f16x2_f32x2_distinct_adds_param_2];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd5, %rd1, %rd4;
+; CHECK-NEXT:    mov.b64 {%r1, _}, %rd3;
+; CHECK-NEXT:    mov.b64 {_, %r2}, %rd5;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum1 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %sum2 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %c, i32 0)
+  %lo = extractelement <2 x float> %sum1, i32 0
+  %hi = extractelement <2 x float> %sum2, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+;
+;  BF16x2
+;
+
+; multiple uses
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2_extra_use(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_extra_use_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_extra_use_param_1];
+; CHECK-NEXT:    add.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    ld.param::func.b64 %rd4, [add_rz_bf16x2_f32x2_extra_use_param_2];
+; CHECK-NEXT:    st.b64 [%rd4], %rd3;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  store <2 x float> %sum, ptr %p
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
+
+; unsupported forms
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2_swapped_halves(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_swapped_halves_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_swapped_halves_param_1];
+; CHECK-NEXT:    add.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %lo, float %hi, i1 false)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @add_rz_bf16x2_f32x2_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rz_bf16x2_f32x2_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_ftz_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_ftz_param_1];
+; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @add_rn_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: add_rn_bf16x2_f32x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rn_bf16x2_f32x2_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rn_bf16x2_f32x2_param_1];
+; CHECK-NEXT:    add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert.ll
index 9dbfe1e3ab014..de26b06fcbdf2 100644
--- a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert.ll
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-downconvert.ll
@@ -44,161 +44,6 @@ define <2 x half> @add_rz_f16x2_f32x2_commuted(<2 x float> %a, <2 x float> %b) {
   ret <2 x half> %r
 }
 
-define <2 x half> @add_rz_f16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
-; CHECK-LABEL: add_rz_f16x2_f32x2_extra_use(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_extra_use_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_extra_use_param_1];
-; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    ld.param::func.b64 %rd4, [add_rz_f16x2_f32x2_extra_use_param_2];
-; CHECK-NEXT:    st.b64 [%rd4], %rd3;
-; CHECK-NEXT:    add.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd1, %rd2;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
-; CHECK-NEXT:    ret;
-  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
-  store <2 x float> %sum, ptr %p
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x half> %r
-}
-
-; invalid patterns
-
-define <2 x half> @add_rz_f16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: add_rz_f16x2_f32x2_swapped_halves(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_swapped_halves_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_swapped_halves_param_1];
-; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r1, %r2;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %lo, float %hi, i1 false)
-  ret <2 x half> %r
-}
-
-define <2 x half> @add_rz_f16x2_f32x2_no_ftz(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: add_rz_f16x2_f32x2_no_ftz(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_no_ftz_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_no_ftz_param_1];
-; CHECK-NEXT:    add.rz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x half> %r
-}
-
-define <2 x half> @add_rn_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: add_rn_f16x2_f32x2(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rn_f16x2_f32x2_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rn_f16x2_f32x2_param_1];
-; CHECK-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x half> %r
-}
-
-define <2 x half> @add_rz_f16x2_f32x2_rn_convert(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: add_rz_f16x2_f32x2_rn_convert(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_rn_convert_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_rn_convert_param_1];
-; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rn.f16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %hi, float %lo, i1 false)
-  ret <2 x half> %r
-}
-
-define <2 x half> @add_rz_f16x2_f32x2_relu(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: add_rz_f16x2_f32x2_relu(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_relu_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_relu_param_1];
-; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.relu.f16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %hi, float %lo, i1 false)
-  ret <2 x half> %r
-}
-
-define <2 x half> @add_rz_f16x2_f32x2_distinct_adds(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
-; CHECK-LABEL: add_rz_f16x2_f32x2_distinct_adds(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<6>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_f16x2_f32x2_distinct_adds_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_f16x2_f32x2_distinct_adds_param_1];
-; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    ld.param::func.b64 %rd4, [add_rz_f16x2_f32x2_distinct_adds_param_2];
-; CHECK-NEXT:    add.rz.ftz.f32x2 %rd5, %rd1, %rd4;
-; CHECK-NEXT:    mov.b64 {%r1, _}, %rd3;
-; CHECK-NEXT:    mov.b64 {_, %r2}, %rd5;
-; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %sum1 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
-  %sum2 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %c, i32 0)
-  %lo = extractelement <2 x float> %sum1, i32 0
-  %hi = extractelement <2 x float> %sum2, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x half> %r
-}
-
 ;
 ;  BF16x2
 ;
@@ -240,91 +85,3 @@ define <2 x bfloat> @add_rz_bf16x2_f32x2_commuted(<2 x float> %a, <2 x float> %b
   %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
   ret <2 x bfloat> %r
 }
-
-define <2 x bfloat> @add_rz_bf16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
-; CHECK-LABEL: add_rz_bf16x2_f32x2_extra_use(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_extra_use_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_extra_use_param_1];
-; CHECK-NEXT:    add.rz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    ld.param::func.b64 %rd4, [add_rz_bf16x2_f32x2_extra_use_param_2];
-; CHECK-NEXT:    st.b64 [%rd4], %rd3;
-; CHECK-NEXT:    add.rz.bf16x2.f32x2.f32x2 %r1, %rd1, %rd2;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
-; CHECK-NEXT:    ret;
-  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
-  store <2 x float> %sum, ptr %p
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x bfloat> %r
-}
-
-; invalid patterns
-
-define <2 x bfloat> @add_rz_bf16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: add_rz_bf16x2_f32x2_swapped_halves(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_swapped_halves_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_swapped_halves_param_1];
-; CHECK-NEXT:    add.rz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r1, %r2;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %lo, float %hi, i1 false)
-  ret <2 x bfloat> %r
-}
-
-define <2 x bfloat> @add_rz_bf16x2_f32x2_ftz(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: add_rz_bf16x2_f32x2_ftz(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rz_bf16x2_f32x2_ftz_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rz_bf16x2_f32x2_ftz_param_1];
-; CHECK-NEXT:    add.rz.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %b, i32 0)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x bfloat> %r
-}
-
-define <2 x bfloat> @add_rn_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: add_rn_bf16x2_f32x2(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_rn_bf16x2_f32x2_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [add_rn_bf16x2_f32x2_param_1];
-; CHECK-NEXT:    add.rn.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %b, i32 1)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x bfloat> %r
-}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert-invalid.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert-invalid.ll
new file mode 100644
index 0000000000000..e28efa99e90a1
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert-invalid.ll
@@ -0,0 +1,711 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck --check-prefixes=CHECK-NOFTZ %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | FileCheck --check-prefixes=CHECK-FTZ %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | %ptxas-verify -arch=sm_107f %}
+
+;
+; Cases where the mixed-precision instruction must not be selected.
+;
+
+;
+;  F16x2
+;
+
+; multiple uses of the extension
+
+define <2 x float> @add_f32x2_f16x2_extra_use(<2 x half> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_extra_use(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_f16x2_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_extra_use(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_extra_use_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_f16x2_extra_use_param_2];
+; CHECK-FTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-FTZ-NEXT:    add.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  store <2 x float> %e, ptr %p
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_f16x2_generic_lanes_multiuse(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_lanes_multiuse(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lanes_multiuse_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_lanes_multiuse_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd4, %rd1, %rd3;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_lanes_multiuse(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lanes_multiuse_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_lanes_multiuse_param_1];
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd4, %rd1, %rd3;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %r1 = fadd <2 x float> %e, %b
+  %r2 = fadd <2 x float> %e, %r1
+  ret <2 x float> %r2
+}
+
+define <2 x float> @add_f32x2_f16x2_generic_packed_multiuse(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_packed_multiuse(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_generic_packed_multiuse_param_0];
+; CHECK-NOFTZ-NEXT:    mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_packed_multiuse_param_1];
+; CHECK-NOFTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r3, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r4, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd2, {%r4, %r3};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd4, %rd2, %rd3;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_packed_multiuse(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_generic_packed_multiuse_param_0];
+; CHECK-FTZ-NEXT:    mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_packed_multiuse_param_1];
+; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r4, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd4, %rd2, %rd3;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %x = fmul <2 x half> %a, %a
+  %e = fpext <2 x half> %x to <2 x float>
+  %r1 = fadd <2 x float> %e, %b
+  %r2 = fadd <2 x float> %e, %r1
+  ret <2 x float> %r2
+}
+
+define <2 x float> @add_f32x2_f16x2_generic_both_lanes_extra_use(<2 x half> %a, <2 x float> %b, ptr %p, ptr %q) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_both_lanes_extra_use(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs2;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd4, {%r1, %r2};
+; CHECK-NOFTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT:    st.b32 [%rd3], %r2;
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd5, %rd4, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_both_lanes_extra_use(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_f16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs2;
+; CHECK-FTZ-NEXT:    mov.b64 %rd4, {%r1, %r2};
+; CHECK-FTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT:    st.b32 [%rd3], %r2;
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd5, %rd4, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT:    ret;
+  %a0 = extractelement <2 x half> %a, i32 0
+  %a1 = extractelement <2 x half> %a, i32 1
+  %e0 = fpext half %a0 to float
+  %e1 = fpext half %a1 to float
+  %v0 = insertelement <2 x float> poison, float %e0, i32 0
+  %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+  store float %e0, ptr %p
+  store float %e1, ptr %q
+  %r = fadd <2 x float> %v1, %b
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_sub_f32x2_f16x2_generic_multiuse(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_sub_f32x2_f16x2_generic_multiuse(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_f16x2_generic_multiuse_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_sub_f32x2_f16x2_generic_multiuse_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd5, %rd3, %rd4;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_sub_f32x2_f16x2_generic_multiuse(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_f16x2_generic_multiuse_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_sub_f32x2_f16x2_generic_multiuse_param_1];
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd5, %rd3, %rd4;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %add = fadd <2 x float> %e, %b
+  %sub = fsub <2 x float> %e, %b
+  %r = fadd <2 x float> %add, %sub
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_sub_f32x2_f16x2_multiuse(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_sub_f32x2_f16x2_multiuse(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_f16x2_multiuse_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_sub_f32x2_f16x2_multiuse_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd5, %rd3, %rd4;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_sub_f32x2_f16x2_multiuse(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_f16x2_multiuse_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_sub_f32x2_f16x2_multiuse_param_1];
+; CHECK-FTZ-NEXT:    add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    add.rn.f32x2 %rd5, %rd3, %rd4;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %add = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  %nb = fneg <2 x float> %b
+  %sub = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %add, <2 x float> %sub, i32 1)
+  ret <2 x float> %r
+}
+
+; unsupported forms
+
+define <2 x float> @add_f32x2_f16x2_ftz(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_ftz(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_ftz_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_ftz_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_ftz(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_ftz_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_ftz_param_1];
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_f16x2_partial(<2 x half> %a, float %c, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_partial(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_partial_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b16 %rs1, [add_f32x2_f16x2_partial_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_partial_param_2];
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd2, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_partial(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<2>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_partial_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b16 %rs1, [add_f32x2_f16x2_partial_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_partial_param_2];
+; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r2, %r1};
+; CHECK-FTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %a0 = extractelement <2 x half> %a, i32 0
+  %e0 = fpext half %a0 to float
+  %v0 = insertelement <2 x float> poison, float %e0, i32 0
+  %v1 = insertelement <2 x float> %v0, float %c, i32 1
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %v1, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+;
+;  BF16x2
+;
+
+; multiple uses of the extension
+
+define <2 x float> @add_f32x2_bf16x2_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_extra_use(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_bf16x2_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_extra_use(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_extra_use_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_bf16x2_extra_use_param_2];
+; CHECK-FTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-FTZ-NEXT:    add.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  store <2 x float> %e, ptr %p
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2_generic_lanes_multiuse(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_lanes_multiuse(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lanes_multiuse_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_lanes_multiuse_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd4, %rd1, %rd3;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_lanes_multiuse(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lanes_multiuse_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_lanes_multiuse_param_1];
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd4, %rd1, %rd3;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %r1 = fadd <2 x float> %e, %b
+  %r2 = fadd <2 x float> %e, %r1
+  ret <2 x float> %r2
+}
+
+define <2 x float> @add_f32x2_bf16x2_generic_packed_multiuse(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_packed_multiuse(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_bf16x2_generic_packed_multiuse_param_0];
+; CHECK-NOFTZ-NEXT:    mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_packed_multiuse_param_1];
+; CHECK-NOFTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r3, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r4, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd2, {%r4, %r3};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd4, %rd2, %rd3;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_packed_multiuse(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_bf16x2_generic_packed_multiuse_param_0];
+; CHECK-FTZ-NEXT:    mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_packed_multiuse_param_1];
+; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r3, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r4, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd4, %rd2, %rd3;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %x = fmul <2 x bfloat> %a, %a
+  %e = fpext <2 x bfloat> %x to <2 x float>
+  %r1 = fadd <2 x float> %e, %b
+  %r2 = fadd <2 x float> %e, %r1
+  ret <2 x float> %r2
+}
+
+define <2 x float> @add_f32x2_bf16x2_generic_both_lanes_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p, ptr %q) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_both_lanes_extra_use(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs2;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd4, {%r1, %r2};
+; CHECK-NOFTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT:    st.b32 [%rd3], %r2;
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd5, %rd4, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_both_lanes_extra_use(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_bf16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs2;
+; CHECK-FTZ-NEXT:    mov.b64 %rd4, {%r1, %r2};
+; CHECK-FTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT:    st.b32 [%rd3], %r2;
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd5, %rd4, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT:    ret;
+  %a0 = extractelement <2 x bfloat> %a, i32 0
+  %a1 = extractelement <2 x bfloat> %a, i32 1
+  %e0 = fpext bfloat %a0 to float
+  %e1 = fpext bfloat %a1 to float
+  %v0 = insertelement <2 x float> poison, float %e0, i32 0
+  %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+  store float %e0, ptr %p
+  store float %e1, ptr %q
+  %r = fadd <2 x float> %v1, %b
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_sub_f32x2_bf16x2_generic_multiuse(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_sub_f32x2_bf16x2_generic_multiuse(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_bf16x2_generic_multiuse_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_sub_f32x2_bf16x2_generic_multiuse_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd5, %rd3, %rd4;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_sub_f32x2_bf16x2_generic_multiuse(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_bf16x2_generic_multiuse_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_sub_f32x2_bf16x2_generic_multiuse_param_1];
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd5, %rd3, %rd4;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %add = fadd <2 x float> %e, %b
+  %sub = fsub <2 x float> %e, %b
+  %r = fadd <2 x float> %add, %sub
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_sub_f32x2_bf16x2_multiuse(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_sub_f32x2_bf16x2_multiuse(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_bf16x2_multiuse_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_sub_f32x2_bf16x2_multiuse_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd5, %rd3, %rd4;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_sub_f32x2_bf16x2_multiuse(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_sub_f32x2_bf16x2_multiuse_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_sub_f32x2_bf16x2_multiuse_param_1];
+; CHECK-FTZ-NEXT:    add.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    add.rn.f32x2 %rd5, %rd3, %rd4;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %add = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  %nb = fneg <2 x float> %b
+  %sub = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %add, <2 x float> %sub, i32 1)
+  ret <2 x float> %r
+}
+
+; unsupported forms
+
+define <2 x float> @add_f32x2_bf16x2_ftz(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_ftz(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_ftz_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_ftz_param_1];
+; CHECK-NOFTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_ftz(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_ftz_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_ftz_param_1];
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @add_f32x2_bf16x2_partial(<2 x bfloat> %a, float %c, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_partial(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<2>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_bf16x2_partial_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b16 %rs1, [add_f32x2_bf16x2_partial_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_partial_param_2];
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd2, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_partial(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<2>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_bf16x2_partial_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b16 %rs1, [add_f32x2_bf16x2_partial_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_partial_param_2];
+; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r2, %r1};
+; CHECK-FTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %a0 = extractelement <2 x bfloat> %a, i32 0
+  %e0 = fpext bfloat %a0 to float
+  %v0 = insertelement <2 x float> poison, float %e0, i32 0
+  %v1 = insertelement <2 x float> %v0, float %c, i32 1
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %v1, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll
index 27708d4177786..c1acfe150289f 100644
--- a/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-add-f32x2-upconvert.ll
@@ -10,29 +10,38 @@
 
 ; lanes
 
-define <2 x float> @add_f32x2_f16x2_lanes(<2 x half> %a, <2 x float> %b) {
+define <2 x float> @add_f32x2_f16x2_lanes(<2 x half> %a0, <2 x half> %a1, <2 x half> %a2, <2 x half> %a3, <2 x float> %b) {
 ; CHECK-LABEL: add_f32x2_f16x2_lanes(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b16 %rs<9>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-NEXT:    .reg .b64 %rd<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_lanes_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_lanes_param_1];
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs3, %rs4}, [add_f32x2_f16x2_lanes_param_1];
 ; CHECK-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_lanes_param_4];
 ; CHECK-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
-; CHECK-NEXT:    add.rz.f32x2.f16x2.f32x2 %rd3, %r1, %rd2;
-; CHECK-NEXT:    add.rm.f32x2.f16x2.f32x2 %rd4, %r1, %rd3;
-; CHECK-NEXT:    add.rp.f32x2.f16x2.f32x2 %rd5, %r1, %rd4;
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs5, %rs6}, [add_f32x2_f16x2_lanes_param_2];
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs7, %rs8}, [add_f32x2_f16x2_lanes_param_3];
+; CHECK-NEXT:    mov.b32 %r2, {%rs3, %rs4};
+; CHECK-NEXT:    add.rz.f32x2.f16x2.f32x2 %rd3, %r2, %rd2;
+; CHECK-NEXT:    mov.b32 %r3, {%rs5, %rs6};
+; CHECK-NEXT:    add.rm.f32x2.f16x2.f32x2 %rd4, %r3, %rd3;
+; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT:    add.rp.f32x2.f16x2.f32x2 %rd5, %r4, %rd4;
 ; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd5;
 ; CHECK-NEXT:    ret;
-  %e = fpext <2 x half> %a to <2 x float>
-  %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
-  %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %r1, i32 0)
-  %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %r2, i32 3)
-  %r4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %r3, i32 2)
-  ret <2 x float> %r4
+  %e0 = fpext <2 x half> %a0 to <2 x float>
+  %r0 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e0, <2 x float> %b, i32 1)
+  %e1 = fpext <2 x half> %a1 to <2 x float>
+  %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e1, <2 x float> %r0, i32 0)
+  %e2 = fpext <2 x half> %a2 to <2 x float>
+  %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e2, <2 x float> %r1, i32 3)
+  %e3 = fpext <2 x half> %a3 to <2 x float>
+  %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e3, <2 x float> %r2, i32 2)
+  ret <2 x float> %r3
 }
 
 define <2 x float> @add_f32x2_f16x2_lanes_commuted(<2 x half> %a, <2 x float> %b) {
@@ -54,50 +63,6 @@ define <2 x float> @add_f32x2_f16x2_lanes_commuted(<2 x half> %a, <2 x float> %b
   ret <2 x float> %r
 }
 
-define <2 x float> @add_f32x2_f16x2_extra_use(<2 x half> %a, <2 x float> %b, ptr %p) {
-; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_extra_use(
-; CHECK-NOFTZ:       {
-; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NOFTZ-NEXT:    .reg .b32 %r<4>;
-; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
-; CHECK-NOFTZ-EMPTY:
-; CHECK-NOFTZ-NEXT:  // %bb.0:
-; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_extra_use_param_0];
-; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
-; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
-; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_extra_use_param_1];
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_f16x2_extra_use_param_2];
-; CHECK-NOFTZ-NEXT:    st.b64 [%rd3], %rd1;
-; CHECK-NOFTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
-; CHECK-NOFTZ-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd4, %r3, %rd2;
-; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
-; CHECK-NOFTZ-NEXT:    ret;
-;
-; CHECK-FTZ-LABEL: add_f32x2_f16x2_extra_use(
-; CHECK-FTZ:       {
-; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
-; CHECK-FTZ-NEXT:    .reg .b32 %r<4>;
-; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
-; CHECK-FTZ-EMPTY:
-; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_extra_use_param_0];
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
-; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_extra_use_param_1];
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_f16x2_extra_use_param_2];
-; CHECK-FTZ-NEXT:    st.b64 [%rd3], %rd1;
-; CHECK-FTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
-; CHECK-FTZ-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd4, %r3, %rd2;
-; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
-; CHECK-FTZ-NEXT:    ret;
-  %e = fpext <2 x half> %a to <2 x float>
-  store <2 x float> %e, ptr %p
-  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
-  ret <2 x float> %r
-}
-
 ; packed
 
 define <2 x float> @add_f32x2_f16x2_packed(<2 x half> %a, <2 x float> %b) {
@@ -273,82 +238,51 @@ define <2 x float> @add_f32x2_f16x2_generic_packed(<2 x half> %a, <2 x float> %b
   ret <2 x float> %r
 }
 
-; invalid patterns
+; generic fadd - partially shared extension
 
-define <2 x float> @add_f32x2_f16x2_ftz(<2 x half> %a, <2 x float> %b) {
-; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_ftz(
+define <2 x float> @add_f32x2_f16x2_generic_lane_extra_use(<2 x half> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_generic_lane_extra_use(
 ; CHECK-NOFTZ:       {
 ; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-NOFTZ-EMPTY:
 ; CHECK-NOFTZ-NEXT:  // %bb.0:
-; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_ftz_param_0];
-; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
-; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
-; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_ftz_param_1];
-; CHECK-NOFTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lane_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_lane_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_lane_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2.f16x2.f32x2 %rd3, %r2, %rd1;
 ; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
 ; CHECK-NOFTZ-NEXT:    ret;
 ;
-; CHECK-FTZ-LABEL: add_f32x2_f16x2_ftz(
+; CHECK-FTZ-LABEL: add_f32x2_f16x2_generic_lane_extra_use(
 ; CHECK-FTZ:       {
 ; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
-; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
-; CHECK-FTZ-EMPTY:
-; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_ftz_param_0];
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
-; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_ftz_param_1];
-; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
-; CHECK-FTZ-NEXT:    ret;
-  %e = fpext <2 x half> %a to <2 x float>
-  %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
-  ret <2 x float> %r
-}
-
-define <2 x float> @add_f32x2_f16x2_partial(<2 x half> %a, float %c, <2 x float> %b) {
-; CHECK-NOFTZ-LABEL: add_f32x2_f16x2_partial(
-; CHECK-NOFTZ:       {
-; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<2>;
-; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
-; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
-; CHECK-NOFTZ-EMPTY:
-; CHECK-NOFTZ-NEXT:  // %bb.0:
-; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_partial_param_1];
-; CHECK-NOFTZ-NEXT:    ld.param::func.b16 %rs1, [add_f32x2_f16x2_partial_param_0];
-; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_partial_param_2];
-; CHECK-NOFTZ-NEXT:    mov.b64 %rd2, {%r2, %r1};
-; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
-; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
-; CHECK-NOFTZ-NEXT:    ret;
-;
-; CHECK-FTZ-LABEL: add_f32x2_f16x2_partial(
-; CHECK-FTZ:       {
-; CHECK-FTZ-NEXT:    .reg .b16 %rs<2>;
-; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
-; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_f16x2_partial_param_1];
-; CHECK-FTZ-NEXT:    ld.param::func.b16 %rs1, [add_f32x2_f16x2_partial_param_0];
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_partial_param_2];
-; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r2, %r1};
-; CHECK-FTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
-; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_f16x2_generic_lane_extra_use_param_0];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_f16x2_generic_lane_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_f16x2_generic_lane_extra_use_param_2];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs1;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs2;
+; CHECK-FTZ-NEXT:    mov.b64 %rd3, {%r1, %r2};
+; CHECK-FTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd4, %rd3, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
 ; CHECK-FTZ-NEXT:    ret;
   %a0 = extractelement <2 x half> %a, i32 0
+  %a1 = extractelement <2 x half> %a, i32 1
   %e0 = fpext half %a0 to float
+  %e1 = fpext half %a1 to float
   %v0 = insertelement <2 x float> poison, float %e0, i32 0
-  %v1 = insertelement <2 x float> %v0, float %c, i32 1
-  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %v1, <2 x float> %b, i32 1)
+  %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+  store float %e0, ptr %p
+  %r = fadd <2 x float> %v1, %b
   ret <2 x float> %r
 }
 
@@ -358,29 +292,38 @@ define <2 x float> @add_f32x2_f16x2_partial(<2 x half> %a, float %c, <2 x float>
 
 ; lanes
 
-define <2 x float> @add_f32x2_bf16x2_lanes(<2 x bfloat> %a, <2 x float> %b) {
+define <2 x float> @add_f32x2_bf16x2_lanes(<2 x bfloat> %a0, <2 x bfloat> %a1, <2 x bfloat> %a2, <2 x bfloat> %a3, <2 x float> %b) {
 ; CHECK-LABEL: add_f32x2_bf16x2_lanes(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b16 %rs<9>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-NEXT:    .reg .b64 %rd<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_lanes_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_lanes_param_1];
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs3, %rs4}, [add_f32x2_bf16x2_lanes_param_1];
 ; CHECK-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_lanes_param_4];
 ; CHECK-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
-; CHECK-NEXT:    add.rz.f32x2.bf16x2.f32x2 %rd3, %r1, %rd2;
-; CHECK-NEXT:    add.rm.f32x2.bf16x2.f32x2 %rd4, %r1, %rd3;
-; CHECK-NEXT:    add.rp.f32x2.bf16x2.f32x2 %rd5, %r1, %rd4;
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs5, %rs6}, [add_f32x2_bf16x2_lanes_param_2];
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs7, %rs8}, [add_f32x2_bf16x2_lanes_param_3];
+; CHECK-NEXT:    mov.b32 %r2, {%rs3, %rs4};
+; CHECK-NEXT:    add.rz.f32x2.bf16x2.f32x2 %rd3, %r2, %rd2;
+; CHECK-NEXT:    mov.b32 %r3, {%rs5, %rs6};
+; CHECK-NEXT:    add.rm.f32x2.bf16x2.f32x2 %rd4, %r3, %rd3;
+; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT:    add.rp.f32x2.bf16x2.f32x2 %rd5, %r4, %rd4;
 ; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd5;
 ; CHECK-NEXT:    ret;
-  %e = fpext <2 x bfloat> %a to <2 x float>
-  %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
-  %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %r1, i32 0)
-  %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %r2, i32 3)
-  %r4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %r3, i32 2)
-  ret <2 x float> %r4
+  %e0 = fpext <2 x bfloat> %a0 to <2 x float>
+  %r0 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e0, <2 x float> %b, i32 1)
+  %e1 = fpext <2 x bfloat> %a1 to <2 x float>
+  %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e1, <2 x float> %r0, i32 0)
+  %e2 = fpext <2 x bfloat> %a2 to <2 x float>
+  %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e2, <2 x float> %r1, i32 3)
+  %e3 = fpext <2 x bfloat> %a3 to <2 x float>
+  %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e3, <2 x float> %r2, i32 2)
+  ret <2 x float> %r3
 }
 
 define <2 x float> @add_f32x2_bf16x2_lanes_commuted(<2 x bfloat> %a, <2 x float> %b) {
@@ -402,50 +345,6 @@ define <2 x float> @add_f32x2_bf16x2_lanes_commuted(<2 x bfloat> %a, <2 x float>
   ret <2 x float> %r
 }
 
-define <2 x float> @add_f32x2_bf16x2_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p) {
-; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_extra_use(
-; CHECK-NOFTZ:       {
-; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NOFTZ-NEXT:    .reg .b32 %r<4>;
-; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
-; CHECK-NOFTZ-EMPTY:
-; CHECK-NOFTZ-NEXT:  // %bb.0:
-; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_extra_use_param_0];
-; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
-; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_extra_use_param_1];
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_bf16x2_extra_use_param_2];
-; CHECK-NOFTZ-NEXT:    st.b64 [%rd3], %rd1;
-; CHECK-NOFTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
-; CHECK-NOFTZ-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd4, %r3, %rd2;
-; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
-; CHECK-NOFTZ-NEXT:    ret;
-;
-; CHECK-FTZ-LABEL: add_f32x2_bf16x2_extra_use(
-; CHECK-FTZ:       {
-; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
-; CHECK-FTZ-NEXT:    .reg .b32 %r<4>;
-; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
-; CHECK-FTZ-EMPTY:
-; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_extra_use_param_0];
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
-; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_extra_use_param_1];
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [add_f32x2_bf16x2_extra_use_param_2];
-; CHECK-FTZ-NEXT:    st.b64 [%rd3], %rd1;
-; CHECK-FTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
-; CHECK-FTZ-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd4, %r3, %rd2;
-; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
-; CHECK-FTZ-NEXT:    ret;
-  %e = fpext <2 x bfloat> %a to <2 x float>
-  store <2 x float> %e, ptr %p
-  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
-  ret <2 x float> %r
-}
-
 ; packed
 
 define <2 x float> @add_f32x2_bf16x2_packed(<2 x bfloat> %a, <2 x float> %b) {
@@ -595,81 +494,50 @@ define <2 x float> @add_f32x2_bf16x2_generic_packed(<2 x bfloat> %a, <2 x float>
   ret <2 x float> %r
 }
 
-; invalid patterns
+; generic fadd - partially shared extension
 
-define <2 x float> @add_f32x2_bf16x2_ftz(<2 x bfloat> %a, <2 x float> %b) {
-; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_ftz(
+define <2 x float> @add_f32x2_bf16x2_generic_lane_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_generic_lane_extra_use(
 ; CHECK-NOFTZ:       {
 ; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-NOFTZ-EMPTY:
 ; CHECK-NOFTZ-NEXT:  // %bb.0:
-; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_ftz_param_0];
-; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
-; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_ftz_param_1];
-; CHECK-NOFTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lane_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_lane_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_lane_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    add.rn.f32x2.bf16x2.f32x2 %rd3, %r2, %rd1;
 ; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
 ; CHECK-NOFTZ-NEXT:    ret;
 ;
-; CHECK-FTZ-LABEL: add_f32x2_bf16x2_ftz(
+; CHECK-FTZ-LABEL: add_f32x2_bf16x2_generic_lane_extra_use(
 ; CHECK-FTZ:       {
 ; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
-; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
-; CHECK-FTZ-EMPTY:
-; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_ftz_param_0];
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
-; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_ftz_param_1];
-; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
-; CHECK-FTZ-NEXT:    ret;
-  %e = fpext <2 x bfloat> %a to <2 x float>
-  %r = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %e, <2 x float> %b, i32 1)
-  ret <2 x float> %r
-}
-
-define <2 x float> @add_f32x2_bf16x2_partial(<2 x bfloat> %a, float %c, <2 x float> %b) {
-; CHECK-NOFTZ-LABEL: add_f32x2_bf16x2_partial(
-; CHECK-NOFTZ:       {
-; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<2>;
-; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
-; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
-; CHECK-NOFTZ-EMPTY:
-; CHECK-NOFTZ-NEXT:  // %bb.0:
-; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_bf16x2_partial_param_1];
-; CHECK-NOFTZ-NEXT:    ld.param::func.b16 %rs1, [add_f32x2_bf16x2_partial_param_0];
-; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_partial_param_2];
-; CHECK-NOFTZ-NEXT:    mov.b64 %rd2, {%r2, %r1};
-; CHECK-NOFTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
-; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
-; CHECK-NOFTZ-NEXT:    ret;
-;
-; CHECK-FTZ-LABEL: add_f32x2_bf16x2_partial(
-; CHECK-FTZ:       {
-; CHECK-FTZ-NEXT:    .reg .b16 %rs<2>;
-; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
-; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [add_f32x2_bf16x2_partial_param_1];
-; CHECK-FTZ-NEXT:    ld.param::func.b16 %rs1, [add_f32x2_bf16x2_partial_param_0];
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_partial_param_2];
-; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r2, %r1};
-; CHECK-FTZ-NEXT:    add.rn.f32x2 %rd3, %rd2, %rd1;
-; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [add_f32x2_bf16x2_generic_lane_extra_use_param_0];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [add_f32x2_bf16x2_generic_lane_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [add_f32x2_bf16x2_generic_lane_extra_use_param_2];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs1;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs2;
+; CHECK-FTZ-NEXT:    mov.b64 %rd3, {%r1, %r2};
+; CHECK-FTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT:    add.rn.ftz.f32x2 %rd4, %rd3, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
 ; CHECK-FTZ-NEXT:    ret;
   %a0 = extractelement <2 x bfloat> %a, i32 0
+  %a1 = extractelement <2 x bfloat> %a, i32 1
   %e0 = fpext bfloat %a0 to float
+  %e1 = fpext bfloat %a1 to float
   %v0 = insertelement <2 x float> poison, float %e0, i32 0
-  %v1 = insertelement <2 x float> %v0, float %c, i32 1
-  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %v1, <2 x float> %b, i32 1)
+  %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+  store float %e0, ptr %p
+  %r = fadd <2 x float> %v1, %b
   ret <2 x float> %r
 }
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert-invalid.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert-invalid.ll
new file mode 100644
index 0000000000000..b38860f261007
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert-invalid.ll
@@ -0,0 +1,276 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+
+;
+; Cases where the mixed-precision instruction must not be selected.
+;
+
+;
+;  F16x2
+;
+
+; multiple uses
+
+define <2 x half> @sub_rz_f16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_extra_use(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_extra_use_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_extra_use_param_1];
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    ld.param::func.b64 %rd4, [sub_rz_f16x2_f32x2_extra_use_param_2];
+; CHECK-NEXT:    st.b64 [%rd4], %rd3;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  store <2 x float> %sum, ptr %p
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+; unsupported forms
+
+define <2 x half> @sub_rz_f16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_swapped_halves(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_swapped_halves_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_swapped_halves_param_1];
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %lo, float %hi, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_no_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_no_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_no_ftz_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_no_ftz_param_1];
+; CHECK-NEXT:    sub.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @sub_rn_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rn_f16x2_f32x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rn_f16x2_f32x2_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rn_f16x2_f32x2_param_1];
+; CHECK-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 1)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_rn_convert(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_rn_convert(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_rn_convert_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_rn_convert_param_1];
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rn.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_relu(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_relu(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_relu_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_relu_param_1];
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.relu.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+define <2 x half> @sub_rz_f16x2_f32x2_distinct_subs(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
+; CHECK-LABEL: sub_rz_f16x2_f32x2_distinct_subs(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<6>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_distinct_subs_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_distinct_subs_param_1];
+; CHECK-NEXT:    ld.param::func.b64 %rd3, [sub_rz_f16x2_f32x2_distinct_subs_param_2];
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd5, %rd1, %rd3;
+; CHECK-NEXT:    mov.b64 {%r1, _}, %rd4;
+; CHECK-NEXT:    mov.b64 {_, %r2}, %rd5;
+; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %nc = fneg <2 x float> %c
+  %sum1 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %sum2 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nc, i32 0)
+  %lo = extractelement <2 x float> %sum1, i32 0
+  %hi = extractelement <2 x float> %sum2, i32 1
+  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x half> %r
+}
+
+;
+;  BF16x2
+;
+
+; multiple uses
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2_extra_use(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<5>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_extra_use_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_extra_use_param_1];
+; CHECK-NEXT:    sub.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    ld.param::func.b64 %rd4, [sub_rz_bf16x2_f32x2_extra_use_param_2];
+; CHECK-NEXT:    st.b64 [%rd4], %rd3;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  store <2 x float> %sum, ptr %p
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
+
+; unsupported forms
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2_swapped_halves(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_swapped_halves_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_swapped_halves_param_1];
+; CHECK-NEXT:    sub.rz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r1, %r2;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %lo, float %hi, i1 false)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @sub_rz_bf16x2_f32x2_ftz(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rz_bf16x2_f32x2_ftz(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_ftz_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_ftz_param_1];
+; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @sub_rn_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
+; CHECK-LABEL: sub_rn_bf16x2_f32x2(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b32 %r<4>;
+; CHECK-NEXT:    .reg .b64 %rd<4>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rn_bf16x2_f32x2_param_0];
+; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rn_bf16x2_f32x2_param_1];
+; CHECK-NEXT:    sub.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
+; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
+; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
+; CHECK-NEXT:    ret;
+  %nb = fneg <2 x float> %b
+  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 1)
+  %lo = extractelement <2 x float> %sum, i32 0
+  %hi = extractelement <2 x float> %sum, i32 1
+  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
+  ret <2 x bfloat> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert.ll
index ae71d63dc5da8..246b04173b657 100644
--- a/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert.ll
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-downconvert.ll
@@ -46,169 +46,6 @@ define <2 x half> @sub_rz_f16x2_f32x2_negated_minuend(<2 x float> %a, <2 x float
   ret <2 x half> %r
 }
 
-define <2 x half> @sub_rz_f16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
-; CHECK-LABEL: sub_rz_f16x2_f32x2_extra_use(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_extra_use_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_extra_use_param_1];
-; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    ld.param::func.b64 %rd4, [sub_rz_f16x2_f32x2_extra_use_param_2];
-; CHECK-NEXT:    st.b64 [%rd4], %rd3;
-; CHECK-NEXT:    sub.rz.ftz.f16x2.f32x2.f32x2 %r1, %rd1, %rd2;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
-; CHECK-NEXT:    ret;
-  %nb = fneg <2 x float> %b
-  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
-  store <2 x float> %sum, ptr %p
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x half> %r
-}
-
-; invalid patterns
-
-define <2 x half> @sub_rz_f16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: sub_rz_f16x2_f32x2_swapped_halves(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_swapped_halves_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_swapped_halves_param_1];
-; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r1, %r2;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %nb = fneg <2 x float> %b
-  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %lo, float %hi, i1 false)
-  ret <2 x half> %r
-}
-
-define <2 x half> @sub_rz_f16x2_f32x2_no_ftz(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: sub_rz_f16x2_f32x2_no_ftz(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_no_ftz_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_no_ftz_param_1];
-; CHECK-NEXT:    sub.rz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %nb = fneg <2 x float> %b
-  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x half> %r
-}
-
-define <2 x half> @sub_rn_f16x2_f32x2(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: sub_rn_f16x2_f32x2(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rn_f16x2_f32x2_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rn_f16x2_f32x2_param_1];
-; CHECK-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %nb = fneg <2 x float> %b
-  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 1)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x half> %r
-}
-
-define <2 x half> @sub_rz_f16x2_f32x2_rn_convert(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: sub_rz_f16x2_f32x2_rn_convert(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_rn_convert_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_rn_convert_param_1];
-; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rn.f16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %nb = fneg <2 x float> %b
-  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rn(float %hi, float %lo, i1 false)
-  ret <2 x half> %r
-}
-
-define <2 x half> @sub_rz_f16x2_f32x2_relu(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: sub_rz_f16x2_f32x2_relu(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_relu_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_relu_param_1];
-; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.relu.f16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %nb = fneg <2 x float> %b
-  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz.relu(float %hi, float %lo, i1 false)
-  ret <2 x half> %r
-}
-
-define <2 x half> @sub_rz_f16x2_f32x2_distinct_subs(<2 x float> %a, <2 x float> %b, <2 x float> %c) {
-; CHECK-LABEL: sub_rz_f16x2_f32x2_distinct_subs(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<6>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_f16x2_f32x2_distinct_subs_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_f16x2_f32x2_distinct_subs_param_1];
-; CHECK-NEXT:    ld.param::func.b64 %rd3, [sub_rz_f16x2_f32x2_distinct_subs_param_2];
-; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd4, %rd1, %rd2;
-; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd5, %rd1, %rd3;
-; CHECK-NEXT:    mov.b64 {%r1, _}, %rd4;
-; CHECK-NEXT:    mov.b64 {_, %r2}, %rd5;
-; CHECK-NEXT:    cvt.rz.f16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %nb = fneg <2 x float> %b
-  %nc = fneg <2 x float> %c
-  %sum1 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
-  %sum2 = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nc, i32 0)
-  %lo = extractelement <2 x float> %sum1, i32 0
-  %hi = extractelement <2 x float> %sum2, i32 1
-  %r = call <2 x half> @llvm.nvvm.ff2f16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x half> %r
-}
-
 ;
 ;  BF16x2
 ;
@@ -252,95 +89,3 @@ define <2 x bfloat> @sub_rz_bf16x2_f32x2_negated_minuend(<2 x float> %a, <2 x fl
   %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
   ret <2 x bfloat> %r
 }
-
-define <2 x bfloat> @sub_rz_bf16x2_f32x2_extra_use(<2 x float> %a, <2 x float> %b, ptr %p) {
-; CHECK-LABEL: sub_rz_bf16x2_f32x2_extra_use(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<2>;
-; CHECK-NEXT:    .reg .b64 %rd<5>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_extra_use_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_extra_use_param_1];
-; CHECK-NEXT:    sub.rz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    ld.param::func.b64 %rd4, [sub_rz_bf16x2_f32x2_extra_use_param_2];
-; CHECK-NEXT:    st.b64 [%rd4], %rd3;
-; CHECK-NEXT:    sub.rz.bf16x2.f32x2.f32x2 %r1, %rd1, %rd2;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r1;
-; CHECK-NEXT:    ret;
-  %nb = fneg <2 x float> %b
-  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
-  store <2 x float> %sum, ptr %p
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x bfloat> %r
-}
-
-; invalid patterns
-
-define <2 x bfloat> @sub_rz_bf16x2_f32x2_swapped_halves(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: sub_rz_bf16x2_f32x2_swapped_halves(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_swapped_halves_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_swapped_halves_param_1];
-; CHECK-NEXT:    sub.rz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r1, %r2;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %nb = fneg <2 x float> %b
-  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %lo, float %hi, i1 false)
-  ret <2 x bfloat> %r
-}
-
-define <2 x bfloat> @sub_rz_bf16x2_f32x2_ftz(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: sub_rz_bf16x2_f32x2_ftz(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rz_bf16x2_f32x2_ftz_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rz_bf16x2_f32x2_ftz_param_1];
-; CHECK-NEXT:    sub.rz.ftz.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %nb = fneg <2 x float> %b
-  %sum = call <2 x float> @llvm.nvvm.fadd.ftz.v2f32(<2 x float> %a, <2 x float> %nb, i32 0)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x bfloat> %r
-}
-
-define <2 x bfloat> @sub_rn_bf16x2_f32x2(<2 x float> %a, <2 x float> %b) {
-; CHECK-LABEL: sub_rn_bf16x2_f32x2(
-; CHECK:       {
-; CHECK-NEXT:    .reg .b32 %r<4>;
-; CHECK-NEXT:    .reg .b64 %rd<4>;
-; CHECK-EMPTY:
-; CHECK-NEXT:  // %bb.0:
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_rn_bf16x2_f32x2_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd2, [sub_rn_bf16x2_f32x2_param_1];
-; CHECK-NEXT:    sub.rn.f32x2 %rd3, %rd1, %rd2;
-; CHECK-NEXT:    mov.b64 {%r1, %r2}, %rd3;
-; CHECK-NEXT:    cvt.rz.bf16x2.f32 %r3, %r2, %r1;
-; CHECK-NEXT:    st.param::func.b32 [func_retval0], %r3;
-; CHECK-NEXT:    ret;
-  %nb = fneg <2 x float> %b
-  %sum = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %a, <2 x float> %nb, i32 1)
-  %lo = extractelement <2 x float> %sum, i32 0
-  %hi = extractelement <2 x float> %sum, i32 1
-  %r = call <2 x bfloat> @llvm.nvvm.ff2bf16x2.rz(float %hi, float %lo, i1 false)
-  ret <2 x bfloat> %r
-}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert-invalid.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert-invalid.ll
new file mode 100644
index 0000000000000..daaff74f5a0c0
--- /dev/null
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert-invalid.ll
@@ -0,0 +1,535 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | FileCheck --check-prefixes=CHECK-NOFTZ %s
+; RUN: llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | FileCheck --check-prefixes=CHECK-FTZ %s
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 | %ptxas-verify -arch=sm_107f %}
+; RUN: %if ptxas-sm_107f && ptxas-isa-9.4 %{ llc < %s -mtriple=nvptx64 -mcpu=sm_107f -mattr=+ptx94 -denormal-fp-math=preserve-sign | %ptxas-verify -arch=sm_107f %}
+
+;
+; Cases where the mixed-precision instruction must not be selected.
+;
+
+;
+;  F16x2
+;
+
+; multiple uses of the extension
+
+define <2 x float> @sub_f32x2_f16x2_extra_use(<2 x half> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_extra_use(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_f16x2_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_extra_use(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_extra_use_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_f16x2_extra_use_param_2];
+; CHECK-FTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-FTZ-NEXT:    sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  store <2 x float> %e, ptr %p
+  %nb = fneg <2 x float> %b
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_f16x2_generic_lanes_multiuse(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_lanes_multiuse(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_lanes_multiuse_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_lanes_multiuse_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd4, %rd1, %rd3;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_lanes_multiuse(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_lanes_multiuse_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_lanes_multiuse_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd4, %rd1, %rd3;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %r1 = fsub <2 x float> %e, %b
+  %r2 = fsub <2 x float> %e, %r1
+  ret <2 x float> %r2
+}
+
+define <2 x float> @sub_f32x2_f16x2_generic_packed_multiuse(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_packed_multiuse(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [sub_f32x2_f16x2_generic_packed_multiuse_param_0];
+; CHECK-NOFTZ-NEXT:    mul.rn.f16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_packed_multiuse_param_1];
+; CHECK-NOFTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r3, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r4, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd2, {%r4, %r3};
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd4, %rd2, %rd3;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_packed_multiuse(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [sub_f32x2_f16x2_generic_packed_multiuse_param_0];
+; CHECK-FTZ-NEXT:    mul.rn.ftz.f16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_packed_multiuse_param_1];
+; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r3, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r4, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd4, %rd2, %rd3;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %x = fmul <2 x half> %a, %a
+  %e = fpext <2 x half> %x to <2 x float>
+  %r1 = fsub <2 x float> %e, %b
+  %r2 = fsub <2 x float> %e, %r1
+  ret <2 x float> %r2
+}
+
+define <2 x float> @sub_f32x2_f16x2_generic_both_lanes_extra_use(<2 x half> %a, <2 x float> %b, ptr %p, ptr %q) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_both_lanes_extra_use(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs2;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd4, {%r1, %r2};
+; CHECK-NOFTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT:    st.b32 [%rd3], %r2;
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd5, %rd4, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_both_lanes_extra_use(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_f16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs2;
+; CHECK-FTZ-NEXT:    mov.b64 %rd4, {%r1, %r2};
+; CHECK-FTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT:    st.b32 [%rd3], %r2;
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd5, %rd4, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT:    ret;
+  %a0 = extractelement <2 x half> %a, i32 0
+  %a1 = extractelement <2 x half> %a, i32 1
+  %e0 = fpext half %a0 to float
+  %e1 = fpext half %a1 to float
+  %v0 = insertelement <2 x float> poison, float %e0, i32 0
+  %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+  store float %e0, ptr %p
+  store float %e1, ptr %q
+  %r = fsub <2 x float> %v1, %b
+  ret <2 x float> %r
+}
+
+; unsupported forms
+
+define <2 x float> @sub_f32x2_f16x2_reversed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_reversed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_reversed_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_reversed_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_reversed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_reversed_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_reversed_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %ne = fneg <2 x float> %e
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %ne, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_f16x2_generic_reversed(<2 x half> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_reversed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_reversed_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_reversed_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_reversed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_reversed_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_reversed_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x half> %a to <2 x float>
+  %r = fsub <2 x float> %b, %e
+  ret <2 x float> %r
+}
+
+;
+;  BF16x2
+;
+
+; multiple uses of the extension
+
+define <2 x float> @sub_f32x2_bf16x2_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_extra_use(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_bf16x2_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_extra_use(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_extra_use_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_bf16x2_extra_use_param_2];
+; CHECK-FTZ-NEXT:    st.b64 [%rd3], %rd1;
+; CHECK-FTZ-NEXT:    sub.rn.f32x2 %rd4, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  store <2 x float> %e, ptr %p
+  %nb = fneg <2 x float> %b
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_bf16x2_generic_lanes_multiuse(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_lanes_multiuse(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_lanes_multiuse_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_lanes_multiuse_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd1, %rd2;
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd4, %rd1, %rd3;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_lanes_multiuse(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_lanes_multiuse_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_lanes_multiuse_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd1, %rd2;
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd4, %rd1, %rd3;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %r1 = fsub <2 x float> %e, %b
+  %r2 = fsub <2 x float> %e, %r1
+  ret <2 x float> %r2
+}
+
+define <2 x float> @sub_f32x2_bf16x2_generic_packed_multiuse(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_packed_multiuse(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<5>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.b32 %r1, [sub_f32x2_bf16x2_generic_packed_multiuse_param_0];
+; CHECK-NOFTZ-NEXT:    mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_packed_multiuse_param_1];
+; CHECK-NOFTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r3, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r4, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd2, {%r4, %r3};
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd4, %rd2, %rd3;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_packed_multiuse(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<5>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.b32 %r1, [sub_f32x2_bf16x2_generic_packed_multiuse_param_0];
+; CHECK-FTZ-NEXT:    mul.rn.bf16x2 %r2, %r1, %r1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_packed_multiuse_param_1];
+; CHECK-FTZ-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r3, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r4, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd2, {%r4, %r3};
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd4, %rd2, %rd3;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
+; CHECK-FTZ-NEXT:    ret;
+  %x = fmul <2 x bfloat> %a, %a
+  %e = fpext <2 x bfloat> %x to <2 x float>
+  %r1 = fsub <2 x float> %e, %b
+  %r2 = fsub <2 x float> %e, %r1
+  ret <2 x float> %r2
+}
+
+define <2 x float> @sub_f32x2_bf16x2_generic_both_lanes_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p, ptr %q) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_both_lanes_extra_use(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs2;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd4, {%r1, %r2};
+; CHECK-NOFTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT:    st.b32 [%rd3], %r2;
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd5, %rd4, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_both_lanes_extra_use(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<6>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_0];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_2];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs1;
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_bf16x2_generic_both_lanes_extra_use_param_3];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs2;
+; CHECK-FTZ-NEXT:    mov.b64 %rd4, {%r1, %r2};
+; CHECK-FTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT:    st.b32 [%rd3], %r2;
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd5, %rd4, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd5;
+; CHECK-FTZ-NEXT:    ret;
+  %a0 = extractelement <2 x bfloat> %a, i32 0
+  %a1 = extractelement <2 x bfloat> %a, i32 1
+  %e0 = fpext bfloat %a0 to float
+  %e1 = fpext bfloat %a1 to float
+  %v0 = insertelement <2 x float> poison, float %e0, i32 0
+  %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+  store float %e0, ptr %p
+  store float %e1, ptr %q
+  %r = fsub <2 x float> %v1, %b
+  ret <2 x float> %r
+}
+
+; unsupported forms
+
+define <2 x float> @sub_f32x2_bf16x2_reversed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_reversed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_reversed_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_reversed_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_reversed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_reversed_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_reversed_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %ne = fneg <2 x float> %e
+  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %ne, <2 x float> %b, i32 1)
+  ret <2 x float> %r
+}
+
+define <2 x float> @sub_f32x2_bf16x2_generic_reversed(<2 x bfloat> %a, <2 x float> %b) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_reversed(
+; CHECK-NOFTZ:       {
+; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-NOFTZ-EMPTY:
+; CHECK-NOFTZ-NEXT:  // %bb.0:
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_reversed_param_0];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
+; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_reversed_param_1];
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-NOFTZ-NEXT:    ret;
+;
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_reversed(
+; CHECK-FTZ:       {
+; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
+; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-EMPTY:
+; CHECK-FTZ-NEXT:  // %bb.0:
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_reversed_param_0];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
+; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_reversed_param_1];
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ret;
+  %e = fpext <2 x bfloat> %a to <2 x float>
+  %r = fsub <2 x float> %b, %e
+  ret <2 x float> %r
+}
diff --git a/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert.ll b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert.ll
index 9872325d3a302..a2f0df199e12d 100644
--- a/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert.ll
+++ b/llvm/test/CodeGen/NVPTX/mixed-precision-sub-f32x2-upconvert.ll
@@ -10,78 +10,42 @@
 
 ; lanes
 
-define <2 x float> @sub_f32x2_f16x2_lanes(<2 x half> %a, <2 x float> %b) {
+define <2 x float> @sub_f32x2_f16x2_lanes(<2 x half> %a0, <2 x half> %a1, <2 x half> %a2, <2 x half> %a3, <2 x float> %b) {
 ; CHECK-LABEL: sub_f32x2_f16x2_lanes(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b16 %rs<9>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-NEXT:    .reg .b64 %rd<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_lanes_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_lanes_param_1];
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs3, %rs4}, [sub_f32x2_f16x2_lanes_param_1];
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs5, %rs6}, [sub_f32x2_f16x2_lanes_param_2];
 ; CHECK-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_lanes_param_4];
 ; CHECK-NEXT:    sub.rn.f32x2.f16x2.f32x2 %rd2, %r1, %rd1;
-; CHECK-NEXT:    sub.rz.f32x2.f16x2.f32x2 %rd3, %r1, %rd2;
-; CHECK-NEXT:    sub.rm.f32x2.f16x2.f32x2 %rd4, %r1, %rd3;
-; CHECK-NEXT:    sub.rp.f32x2.f16x2.f32x2 %rd5, %r1, %rd4;
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs7, %rs8}, [sub_f32x2_f16x2_lanes_param_3];
+; CHECK-NEXT:    mov.b32 %r2, {%rs3, %rs4};
+; CHECK-NEXT:    sub.rz.f32x2.f16x2.f32x2 %rd3, %r2, %rd2;
+; CHECK-NEXT:    mov.b32 %r3, {%rs5, %rs6};
+; CHECK-NEXT:    sub.rm.f32x2.f16x2.f32x2 %rd4, %r3, %rd3;
+; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT:    sub.rp.f32x2.f16x2.f32x2 %rd5, %r4, %rd4;
 ; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd5;
 ; CHECK-NEXT:    ret;
-  %e = fpext <2 x half> %a to <2 x float>
-  %nb = fneg <2 x float> %b
-  %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
-  %n1 = fneg <2 x float> %r1
-  %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %n1, i32 0)
-  %n2 = fneg <2 x float> %r2
-  %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %n2, i32 3)
-  %n3 = fneg <2 x float> %r3
-  %r4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %n3, i32 2)
-  ret <2 x float> %r4
-}
-
-define <2 x float> @sub_f32x2_f16x2_extra_use(<2 x half> %a, <2 x float> %b, ptr %p) {
-; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_extra_use(
-; CHECK-NOFTZ:       {
-; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NOFTZ-NEXT:    .reg .b32 %r<4>;
-; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
-; CHECK-NOFTZ-EMPTY:
-; CHECK-NOFTZ-NEXT:  // %bb.0:
-; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_extra_use_param_0];
-; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
-; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
-; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_extra_use_param_1];
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_f16x2_extra_use_param_2];
-; CHECK-NOFTZ-NEXT:    st.b64 [%rd3], %rd1;
-; CHECK-NOFTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
-; CHECK-NOFTZ-NEXT:    sub.rn.f32x2.f16x2.f32x2 %rd4, %r3, %rd2;
-; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
-; CHECK-NOFTZ-NEXT:    ret;
-;
-; CHECK-FTZ-LABEL: sub_f32x2_f16x2_extra_use(
-; CHECK-FTZ:       {
-; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
-; CHECK-FTZ-NEXT:    .reg .b32 %r<4>;
-; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
-; CHECK-FTZ-EMPTY:
-; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_extra_use_param_0];
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
-; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_extra_use_param_1];
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_f16x2_extra_use_param_2];
-; CHECK-FTZ-NEXT:    st.b64 [%rd3], %rd1;
-; CHECK-FTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
-; CHECK-FTZ-NEXT:    sub.rn.f32x2.f16x2.f32x2 %rd4, %r3, %rd2;
-; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
-; CHECK-FTZ-NEXT:    ret;
-  %e = fpext <2 x half> %a to <2 x float>
-  store <2 x float> %e, ptr %p
-  %nb = fneg <2 x float> %b
-  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
-  ret <2 x float> %r
+  %e0 = fpext <2 x half> %a0 to <2 x float>
+  %n0 = fneg <2 x float> %b
+  %r0 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e0, <2 x float> %n0, i32 1)
+  %e1 = fpext <2 x half> %a1 to <2 x float>
+  %n1 = fneg <2 x float> %r0
+  %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e1, <2 x float> %n1, i32 0)
+  %e2 = fpext <2 x half> %a2 to <2 x float>
+  %n2 = fneg <2 x float> %r1
+  %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e2, <2 x float> %n2, i32 3)
+  %e3 = fpext <2 x half> %a3 to <2 x float>
+  %n3 = fneg <2 x float> %r2
+  %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e3, <2 x float> %n3, i32 2)
+  ret <2 x float> %r3
 }
 
 ; packed
@@ -193,80 +157,51 @@ define <2 x float> @sub_f32x2_f16x2_generic_packed(<2 x half> %a, <2 x float> %b
   ret <2 x float> %r
 }
 
-; invalid patterns
+; generic fsub - partially shared extension
 
-define <2 x float> @sub_f32x2_f16x2_reversed(<2 x half> %a, <2 x float> %b) {
-; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_reversed(
+define <2 x float> @sub_f32x2_f16x2_generic_lane_extra_use(<2 x half> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_lane_extra_use(
 ; CHECK-NOFTZ:       {
 ; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-NOFTZ-EMPTY:
 ; CHECK-NOFTZ-NEXT:  // %bb.0:
-; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_reversed_param_0];
-; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
-; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
-; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_reversed_param_1];
-; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_lane_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_lane_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_lane_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2.f16x2.f32x2 %rd3, %r2, %rd1;
 ; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
 ; CHECK-NOFTZ-NEXT:    ret;
 ;
-; CHECK-FTZ-LABEL: sub_f32x2_f16x2_reversed(
+; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_lane_extra_use(
 ; CHECK-FTZ:       {
 ; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
-; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
-; CHECK-FTZ-EMPTY:
-; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_reversed_param_0];
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
-; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_reversed_param_1];
-; CHECK-FTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
-; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
-; CHECK-FTZ-NEXT:    ret;
-  %e = fpext <2 x half> %a to <2 x float>
-  %ne = fneg <2 x float> %e
-  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %ne, <2 x float> %b, i32 1)
-  ret <2 x float> %r
-}
-
-define <2 x float> @sub_f32x2_f16x2_generic_reversed(<2 x half> %a, <2 x float> %b) {
-; CHECK-NOFTZ-LABEL: sub_f32x2_f16x2_generic_reversed(
-; CHECK-NOFTZ:       {
-; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
-; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
-; CHECK-NOFTZ-EMPTY:
-; CHECK-NOFTZ-NEXT:  // %bb.0:
-; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_reversed_param_0];
-; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r1, %rs2;
-; CHECK-NOFTZ-NEXT:    cvt.f32.f16 %r2, %rs1;
-; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_reversed_param_1];
-; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
-; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
-; CHECK-NOFTZ-NEXT:    ret;
-;
-; CHECK-FTZ-LABEL: sub_f32x2_f16x2_generic_reversed(
-; CHECK-FTZ:       {
-; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
-; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
-; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_reversed_param_0];
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs2;
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs1;
-; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_reversed_param_1];
-; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
-; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_f16x2_generic_lane_extra_use_param_0];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_f16x2_generic_lane_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_f16x2_generic_lane_extra_use_param_2];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r1, %rs1;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.f16 %r2, %rs2;
+; CHECK-FTZ-NEXT:    mov.b64 %rd3, {%r1, %r2};
+; CHECK-FTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd4, %rd3, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
 ; CHECK-FTZ-NEXT:    ret;
-  %e = fpext <2 x half> %a to <2 x float>
-  %r = fsub <2 x float> %b, %e
+  %a0 = extractelement <2 x half> %a, i32 0
+  %a1 = extractelement <2 x half> %a, i32 1
+  %e0 = fpext half %a0 to float
+  %e1 = fpext half %a1 to float
+  %v0 = insertelement <2 x float> poison, float %e0, i32 0
+  %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+  store float %e0, ptr %p
+  %r = fsub <2 x float> %v1, %b
   ret <2 x float> %r
 }
 
@@ -276,78 +211,42 @@ define <2 x float> @sub_f32x2_f16x2_generic_reversed(<2 x half> %a, <2 x float>
 
 ; lanes
 
-define <2 x float> @sub_f32x2_bf16x2_lanes(<2 x bfloat> %a, <2 x float> %b) {
+define <2 x float> @sub_f32x2_bf16x2_lanes(<2 x bfloat> %a0, <2 x bfloat> %a1, <2 x bfloat> %a2, <2 x bfloat> %a3, <2 x float> %b) {
 ; CHECK-LABEL: sub_f32x2_bf16x2_lanes(
 ; CHECK:       {
-; CHECK-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NEXT:    .reg .b32 %r<2>;
+; CHECK-NEXT:    .reg .b16 %rs<9>;
+; CHECK-NEXT:    .reg .b32 %r<5>;
 ; CHECK-NEXT:    .reg .b64 %rd<6>;
 ; CHECK-EMPTY:
 ; CHECK-NEXT:  // %bb.0:
 ; CHECK-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_lanes_param_0];
-; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_lanes_param_1];
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs3, %rs4}, [sub_f32x2_bf16x2_lanes_param_1];
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs5, %rs6}, [sub_f32x2_bf16x2_lanes_param_2];
 ; CHECK-NEXT:    mov.b32 %r1, {%rs1, %rs2};
+; CHECK-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_lanes_param_4];
 ; CHECK-NEXT:    sub.rn.f32x2.bf16x2.f32x2 %rd2, %r1, %rd1;
-; CHECK-NEXT:    sub.rz.f32x2.bf16x2.f32x2 %rd3, %r1, %rd2;
-; CHECK-NEXT:    sub.rm.f32x2.bf16x2.f32x2 %rd4, %r1, %rd3;
-; CHECK-NEXT:    sub.rp.f32x2.bf16x2.f32x2 %rd5, %r1, %rd4;
+; CHECK-NEXT:    ld.param::func.v2.b16 {%rs7, %rs8}, [sub_f32x2_bf16x2_lanes_param_3];
+; CHECK-NEXT:    mov.b32 %r2, {%rs3, %rs4};
+; CHECK-NEXT:    sub.rz.f32x2.bf16x2.f32x2 %rd3, %r2, %rd2;
+; CHECK-NEXT:    mov.b32 %r3, {%rs5, %rs6};
+; CHECK-NEXT:    sub.rm.f32x2.bf16x2.f32x2 %rd4, %r3, %rd3;
+; CHECK-NEXT:    mov.b32 %r4, {%rs7, %rs8};
+; CHECK-NEXT:    sub.rp.f32x2.bf16x2.f32x2 %rd5, %r4, %rd4;
 ; CHECK-NEXT:    st.param::func.b64 [func_retval0], %rd5;
 ; CHECK-NEXT:    ret;
-  %e = fpext <2 x bfloat> %a to <2 x float>
-  %nb = fneg <2 x float> %b
-  %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
-  %n1 = fneg <2 x float> %r1
-  %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %n1, i32 0)
-  %n2 = fneg <2 x float> %r2
-  %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %n2, i32 3)
-  %n3 = fneg <2 x float> %r3
-  %r4 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %n3, i32 2)
-  ret <2 x float> %r4
-}
-
-define <2 x float> @sub_f32x2_bf16x2_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p) {
-; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_extra_use(
-; CHECK-NOFTZ:       {
-; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NOFTZ-NEXT:    .reg .b32 %r<4>;
-; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<5>;
-; CHECK-NOFTZ-EMPTY:
-; CHECK-NOFTZ-NEXT:  // %bb.0:
-; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_extra_use_param_0];
-; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
-; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_extra_use_param_1];
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_bf16x2_extra_use_param_2];
-; CHECK-NOFTZ-NEXT:    st.b64 [%rd3], %rd1;
-; CHECK-NOFTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
-; CHECK-NOFTZ-NEXT:    sub.rn.f32x2.bf16x2.f32x2 %rd4, %r3, %rd2;
-; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
-; CHECK-NOFTZ-NEXT:    ret;
-;
-; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_extra_use(
-; CHECK-FTZ:       {
-; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
-; CHECK-FTZ-NEXT:    .reg .b32 %r<4>;
-; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
-; CHECK-FTZ-EMPTY:
-; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_extra_use_param_0];
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
-; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_extra_use_param_1];
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd3, [sub_f32x2_bf16x2_extra_use_param_2];
-; CHECK-FTZ-NEXT:    st.b64 [%rd3], %rd1;
-; CHECK-FTZ-NEXT:    mov.b32 %r3, {%rs1, %rs2};
-; CHECK-FTZ-NEXT:    sub.rn.f32x2.bf16x2.f32x2 %rd4, %r3, %rd2;
-; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
-; CHECK-FTZ-NEXT:    ret;
-  %e = fpext <2 x bfloat> %a to <2 x float>
-  store <2 x float> %e, ptr %p
-  %nb = fneg <2 x float> %b
-  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e, <2 x float> %nb, i32 1)
-  ret <2 x float> %r
+  %e0 = fpext <2 x bfloat> %a0 to <2 x float>
+  %n0 = fneg <2 x float> %b
+  %r0 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e0, <2 x float> %n0, i32 1)
+  %e1 = fpext <2 x bfloat> %a1 to <2 x float>
+  %n1 = fneg <2 x float> %r0
+  %r1 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e1, <2 x float> %n1, i32 0)
+  %e2 = fpext <2 x bfloat> %a2 to <2 x float>
+  %n2 = fneg <2 x float> %r1
+  %r2 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e2, <2 x float> %n2, i32 3)
+  %e3 = fpext <2 x bfloat> %a3 to <2 x float>
+  %n3 = fneg <2 x float> %r2
+  %r3 = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %e3, <2 x float> %n3, i32 2)
+  ret <2 x float> %r3
 }
 
 ; packed
@@ -446,79 +345,50 @@ define <2 x float> @sub_f32x2_bf16x2_generic_packed(<2 x bfloat> %a, <2 x float>
   ret <2 x float> %r
 }
 
-; invalid patterns
+; generic fsub - partially shared extension
 
-define <2 x float> @sub_f32x2_bf16x2_reversed(<2 x bfloat> %a, <2 x float> %b) {
-; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_reversed(
+define <2 x float> @sub_f32x2_bf16x2_generic_lane_extra_use(<2 x bfloat> %a, <2 x float> %b, ptr %p) {
+; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_lane_extra_use(
 ; CHECK-NOFTZ:       {
 ; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
 ; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
 ; CHECK-NOFTZ-EMPTY:
 ; CHECK-NOFTZ-NEXT:  // %bb.0:
-; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_reversed_param_0];
-; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
-; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_reversed_param_1];
-; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
+; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_lane_extra_use_param_0];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_lane_extra_use_param_1];
+; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_lane_extra_use_param_2];
+; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs1;
+; CHECK-NOFTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-NOFTZ-NEXT:    mov.b32 %r2, {%rs1, %rs2};
+; CHECK-NOFTZ-NEXT:    sub.rn.f32x2.bf16x2.f32x2 %rd3, %r2, %rd1;
 ; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
 ; CHECK-NOFTZ-NEXT:    ret;
 ;
-; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_reversed(
+; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_lane_extra_use(
 ; CHECK-FTZ:       {
 ; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
 ; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
-; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
-; CHECK-FTZ-EMPTY:
-; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_reversed_param_0];
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
-; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_reversed_param_1];
-; CHECK-FTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
-; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
-; CHECK-FTZ-NEXT:    ret;
-  %e = fpext <2 x bfloat> %a to <2 x float>
-  %ne = fneg <2 x float> %e
-  %r = call <2 x float> @llvm.nvvm.fadd.v2f32(<2 x float> %ne, <2 x float> %b, i32 1)
-  ret <2 x float> %r
-}
-
-define <2 x float> @sub_f32x2_bf16x2_generic_reversed(<2 x bfloat> %a, <2 x float> %b) {
-; CHECK-NOFTZ-LABEL: sub_f32x2_bf16x2_generic_reversed(
-; CHECK-NOFTZ:       {
-; CHECK-NOFTZ-NEXT:    .reg .b16 %rs<3>;
-; CHECK-NOFTZ-NEXT:    .reg .b32 %r<3>;
-; CHECK-NOFTZ-NEXT:    .reg .b64 %rd<4>;
-; CHECK-NOFTZ-EMPTY:
-; CHECK-NOFTZ-NEXT:  // %bb.0:
-; CHECK-NOFTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_reversed_param_0];
-; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r1, %rs2;
-; CHECK-NOFTZ-NEXT:    cvt.f32.bf16 %r2, %rs1;
-; CHECK-NOFTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-NOFTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_reversed_param_1];
-; CHECK-NOFTZ-NEXT:    sub.rn.f32x2 %rd3, %rd2, %rd1;
-; CHECK-NOFTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
-; CHECK-NOFTZ-NEXT:    ret;
-;
-; CHECK-FTZ-LABEL: sub_f32x2_bf16x2_generic_reversed(
-; CHECK-FTZ:       {
-; CHECK-FTZ-NEXT:    .reg .b16 %rs<3>;
-; CHECK-FTZ-NEXT:    .reg .b32 %r<3>;
-; CHECK-FTZ-NEXT:    .reg .b64 %rd<4>;
+; CHECK-FTZ-NEXT:    .reg .b64 %rd<5>;
 ; CHECK-FTZ-EMPTY:
 ; CHECK-FTZ-NEXT:  // %bb.0:
-; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_reversed_param_0];
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs2;
-; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs1;
-; CHECK-FTZ-NEXT:    mov.b64 %rd1, {%r2, %r1};
-; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_reversed_param_1];
-; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd3, %rd2, %rd1;
-; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd3;
+; CHECK-FTZ-NEXT:    ld.param::func.v2.b16 {%rs1, %rs2}, [sub_f32x2_bf16x2_generic_lane_extra_use_param_0];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd1, [sub_f32x2_bf16x2_generic_lane_extra_use_param_1];
+; CHECK-FTZ-NEXT:    ld.param::func.b64 %rd2, [sub_f32x2_bf16x2_generic_lane_extra_use_param_2];
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r1, %rs1;
+; CHECK-FTZ-NEXT:    cvt.ftz.f32.bf16 %r2, %rs2;
+; CHECK-FTZ-NEXT:    mov.b64 %rd3, {%r1, %r2};
+; CHECK-FTZ-NEXT:    st.b32 [%rd2], %r1;
+; CHECK-FTZ-NEXT:    sub.rn.ftz.f32x2 %rd4, %rd3, %rd1;
+; CHECK-FTZ-NEXT:    st.param::func.b64 [func_retval0], %rd4;
 ; CHECK-FTZ-NEXT:    ret;
-  %e = fpext <2 x bfloat> %a to <2 x float>
-  %r = fsub <2 x float> %b, %e
+  %a0 = extractelement <2 x bfloat> %a, i32 0
+  %a1 = extractelement <2 x bfloat> %a, i32 1
+  %e0 = fpext bfloat %a0 to float
+  %e1 = fpext bfloat %a1 to float
+  %v0 = insertelement <2 x float> poison, float %e0, i32 0
+  %v1 = insertelement <2 x float> %v0, float %e1, i32 1
+  store float %e0, ptr %p
+  %r = fsub <2 x float> %v1, %b
   ret <2 x float> %r
 }

>From 067fd165c906cc9e95e7cda038147d48f2df6da4 Mon Sep 17 00:00:00 2001
From: Srinivasa Ravi <srinivasar at nvidia.com>
Date: Thu, 10 Sep 2026 05:45:37 +0000
Subject: [PATCH 7/7] address comments

---
 llvm/lib/Target/NVPTX/NVPTXIntrinsics.td | 114 +++++++++++++----------
 1 file changed, 64 insertions(+), 50 deletions(-)

diff --git a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
index 424f4a80be890..4ef4017c64422 100644
--- a/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
+++ b/llvm/lib/Target/NVPTX/NVPTXIntrinsics.td
@@ -2494,9 +2494,9 @@ foreach ftz = ["", "ftz"] in {
 def INT_NVVM_ADD_D :
   F_MATH_2_RNDOP_TY<"add.${rnd}.f64", F64RT, int_nvvm_fadd>;
 
-// mixed precision
+// Mixed Precision
 
-// match only when the fold removes at least one explicit conversion, and the
+// Match only when the fold removes at least one explicit conversion, and the
 // extended vector is not shared.
 class FoldableFPExtendV2F32<dag ops, dag frag>
   : PatFrag<ops, frag, [{
@@ -2504,7 +2504,7 @@ class FoldableFPExtendV2F32<dag ops, dag frag>
                                 N->getOperand(1)->hasOneUse());
     }]>;
 
-// FP_EXTEND to v2f32 is scalarized before isel, leaving one of two shapes
+// FP_EXTEND to v2f32 is scalarized before ISEL, leaving one of two patterns
 // behind depending on whether the source lanes share a register.
 def fpextend_v2f32_from_packed
   : FoldableFPExtendV2F32<(ops node:$a),
@@ -2518,7 +2518,7 @@ def fpextend_v2f32_from_lanes
                             (f32 (fpextend node:$a0)),
                             (f32 (fpextend node:$a1))))>;
 
-// fuse downconverting add/sub only when the result of the add/sub is solely
+// Fuse downconverting add/sub only when the result of the add/sub is solely
 // used for downconverting.
 def extractelt_only_for_cvt
   : PatFrag<(ops node:$vec, node:$lane), (extractelt node:$vec, node:$lane),
@@ -2542,17 +2542,20 @@ foreach rnd = FPRoundingModes in {
     foreach t = [F16X2RT, BF16X2RT] in {
       def INT_NVVM_MIXED_ADD_ # rnd # _f32x2_ # t.PtxType :
         BasicNVPTXInst<(outs B64:$dst), (ins B32:$a, B64:$b),
-          "add." # rnd # ".f32x2." # t.PtxType # ".f32x2",
-          [(set v2f32:$dst,
-            (int_nvvm_fadd (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b,
-                          rnd_imm))]>;
+          "add." # rnd # ".f32x2." # t.PtxType # ".f32x2">;
+
+      defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_ADD_" # rnd #
+                                       "_f32x2_" # t.PtxType);
+
+      def : Pat<(v2f32 (int_nvvm_fadd (fpextend_v2f32_from_packed t.Ty:$a),
+                                      v2f32:$b, rnd_imm)),
+                (Inst $a, $b)>;
 
       def : Pat<(v2f32 (int_nvvm_fadd
                         (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
                                                     t.Ty.ElementType:$a1),
                         v2f32:$b, rnd_imm)),
-                (!cast<Instruction>("INT_NVVM_MIXED_ADD_" # rnd # "_f32x2_" #
-                                    t.PtxType) (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
+                (Inst (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
     }
   }
 }
@@ -2579,27 +2582,31 @@ let Predicates = [hasRubinFamilySupport, doNoF32FTZ] in
   }
 
 
-// mixed precision - downconverting
+// Mixed precision - Downconverting
 
 // f16x2 - rz rounding mode, only with ftz
 // bf16x2 - rz rounding mode, without ftz
-foreach t = [F16X2RT, BF16X2RT] in {
-  defvar ftz = !if(!eq(t, F16X2RT), ".ftz", "");
-  defvar AddOp = !cast<Intrinsic>("int_nvvm_fadd" # !if(!eq(ftz, ""), "", "_ftz"));
-  defvar CvtOp = !cast<Intrinsic>("int_nvvm_ff2" # t.PtxType # "_rz");
-
-  def INT_NVVM_MIXED_ADD_rz_ # t.PtxType # _f32x2 :
-    BasicNVPTXInst<(outs B32:$dst), (ins B64:$a, B64:$b),
-      "add.rz" # ftz # "." # t.PtxType # ".f32x2.f32x2",
-      // The cvt packs its first argument into the high half of the result.
-      [(set t.Ty:$dst,
-        (CvtOp (extractelt_only_for_cvt
-                 (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 1),
-               (extractelt_only_for_cvt
-                 (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 0),
-               /*pzo=*/0))]>,
-    Requires<[hasRubinFamilySupport]>;
-}
+let Predicates = [hasRubinFamilySupport] in
+  foreach t = [F16X2RT, BF16X2RT] in {
+    defvar ftz = !if(!eq(t, F16X2RT), ".ftz", "");
+    defvar AddOp = !cast<Intrinsic>("int_nvvm_fadd" # !if(!eq(ftz, ""), "", "_ftz"));
+    defvar CvtOp = !cast<Intrinsic>("int_nvvm_ff2" # t.PtxType # "_rz");
+
+    def INT_NVVM_MIXED_ADD_rz_ # t.PtxType # _f32x2 :
+      BasicNVPTXInst<(outs B32:$dst), (ins B64:$a, B64:$b),
+        "add.rz" # ftz # "." # t.PtxType # ".f32x2.f32x2">;
+
+    defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_ADD_rz_" # t.PtxType #
+                                     "_f32x2");
+
+    // The cvt packs its first argument into the upper half of the result.
+    def : Pat<(CvtOp (extractelt_only_for_cvt
+                       (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 1),
+                     (extractelt_only_for_cvt
+                       (AddOp v2f32:$a, v2f32:$b, fp_rnd_rz_imm), 0),
+                     /*pzo=*/0),
+              (Inst $a, $b)>;
+  }
 
 //
 // Sub
@@ -2661,7 +2668,7 @@ def INT_NVVM_SUB_D :
     (ins FPRoundingMode:$rnd), "sub.${rnd}.f64",
     [(set f64:$dst, (int_nvvm_fadd f64:$a, (f64 (fneg f64:$b)), timm:$rnd))]>;
 
-// mixed precision
+// Mixed Precision
 
 foreach rnd = FPRoundingModes in {
   defvar rnd_imm = !cast<TImmLeaf>("fp_rnd_" # rnd # "_imm");
@@ -2684,16 +2691,19 @@ foreach rnd = FPRoundingModes in {
 
       def INT_NVVM_MIXED_SUB_ # rnd # _f32x2_ # t.PtxType :
         BasicNVPTXInst<(outs B64:$dst), (ins B32:$a, B64:$b),
-          "sub." # rnd # ".f32x2." # t.PtxType # ".f32x2",
-          [(set v2f32:$dst,
-            (SubOp (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b))]>;
+          "sub." # rnd # ".f32x2." # t.PtxType # ".f32x2">;
+
+      defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_SUB_" # rnd #
+                                       "_f32x2_" # t.PtxType);
+
+      def : Pat<(v2f32 (SubOp (fpextend_v2f32_from_packed t.Ty:$a), v2f32:$b)),
+                (Inst $a, $b)>;
 
       def : Pat<(v2f32 (SubOp
                         (fpextend_v2f32_from_lanes t.Ty.ElementType:$a0,
                                                     t.Ty.ElementType:$a1),
                         v2f32:$b)),
-                (!cast<Instruction>("INT_NVVM_MIXED_SUB_" # rnd # "_f32x2_" #
-                                    t.PtxType) (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
+                (Inst (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
     }
   }
 }
@@ -2719,25 +2729,29 @@ let Predicates = [hasRubinFamilySupport, doNoF32FTZ] in
               (Inst (t.Ty (V2I16toI32 $a0, $a1)), $b)>;
   }
 
-// mixed precision - downconverting
+// Mixed precision - Downconverting
 
 // f16x2 - rz rounding mode, only with ftz
 // bf16x2 - rz rounding mode, without ftz
-foreach t = [F16X2RT, BF16X2RT] in {
-  defvar ftz = !if(!eq(t, F16X2RT), ".ftz", "");
-  defvar SubOp = !cast<SDNode>("sub_rz" # !if(!eq(ftz, ""), "", "_ftz"));
-  defvar CvtOp = !cast<Intrinsic>("int_nvvm_ff2" # t.PtxType # "_rz");
-
-  def INT_NVVM_MIXED_SUB_rz_ # t.PtxType # _f32x2 :
-    BasicNVPTXInst<(outs B32:$dst), (ins B64:$a, B64:$b),
-      "sub.rz" # ftz # "." # t.PtxType # ".f32x2.f32x2",
-      // The cvt packs its first argument into the high half of the result.
-      [(set t.Ty:$dst,
-        (CvtOp (extractelt_only_for_cvt (SubOp v2f32:$a, v2f32:$b), 1),
-               (extractelt_only_for_cvt (SubOp v2f32:$a, v2f32:$b), 0),
-               /*pzo=*/0))]>,
-    Requires<[hasRubinFamilySupport]>;
-}
+let Predicates = [hasRubinFamilySupport] in
+  foreach t = [F16X2RT, BF16X2RT] in {
+    defvar ftz = !if(!eq(t, F16X2RT), ".ftz", "");
+    defvar SubOp = !cast<SDNode>("sub_rz" # !if(!eq(ftz, ""), "", "_ftz"));
+    defvar CvtOp = !cast<Intrinsic>("int_nvvm_ff2" # t.PtxType # "_rz");
+
+    def INT_NVVM_MIXED_SUB_rz_ # t.PtxType # _f32x2 :
+      BasicNVPTXInst<(outs B32:$dst), (ins B64:$a, B64:$b),
+        "sub.rz" # ftz # "." # t.PtxType # ".f32x2.f32x2">;
+
+    defvar Inst = !cast<Instruction>("INT_NVVM_MIXED_SUB_rz_" # t.PtxType #
+                                     "_f32x2");
+
+    // The cvt packs its first argument into the upper half of the result.
+    def : Pat<(CvtOp (extractelt_only_for_cvt (SubOp v2f32:$a, v2f32:$b), 1),
+                     (extractelt_only_for_cvt (SubOp v2f32:$a, v2f32:$b), 0),
+                     /*pzo=*/0),
+              (Inst $a, $b)>;
+  }
 
 //
 // BFIND

