[llvm] a9ce181 - [NVPTX] Fix bugs involving maximum/minimum and bf16

David Majnemer via llvm-commits llvm-commits at lists.llvm.org
Mon Aug 19 22:10:29 PDT 2024


Author: David Majnemer
Date: 2024-08-20T05:09:49Z
New Revision: a9ce1818174b3e690545b573cc5804d73491ca1a

URL: https://github.com/llvm/llvm-project/commit/a9ce1818174b3e690545b573cc5804d73491ca1a
DIFF: https://github.com/llvm/llvm-project/commit/a9ce1818174b3e690545b573cc5804d73491ca1a.diff

LOG: [NVPTX] Fix bugs involving maximum/minimum and bf16

We would crash on sufficiently old NV hardware (Volta or so) due to
incorrectly marking certain operations legal.

Added: 
    

Modified: 
    llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
    llvm/test/CodeGen/NVPTX/bf16-instructions.ll

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
index 577141299b9496..73791102fc04de 100644
--- a/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
+++ b/llvm/lib/Target/NVPTX/NVPTXISelLowering.cpp
@@ -429,29 +429,50 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
 
   auto setFP16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
                                     LegalizeAction NoF16Action) {
-    setOperationAction(Op, VT, STI.allowFP16Math() ? Action : NoF16Action);
+    bool IsOpSupported = STI.allowFP16Math();
+    switch (Op) {
+    // Several FP16 instructions are available on sm_80 only.
+    case ISD::FMINNUM:
+    case ISD::FMAXNUM:
+    case ISD::FMAXNUM_IEEE:
+    case ISD::FMINNUM_IEEE:
+    case ISD::FMAXIMUM:
+    case ISD::FMINIMUM:
+      IsOpSupported &= STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
+      break;
+    }
+    setOperationAction(Op, VT, IsOpSupported ? Action : NoF16Action);
   };
 
   auto setBF16OperationAction = [&](unsigned Op, MVT VT, LegalizeAction Action,
                                     LegalizeAction NoBF16Action) {
     bool IsOpSupported = STI.hasBF16Math();
-    // Few instructions are available on sm_90 only
-    switch(Op) {
-      case ISD::FADD:
-      case ISD::FMUL:
-      case ISD::FSUB:
-      case ISD::SELECT:
-      case ISD::SELECT_CC:
-      case ISD::SETCC:
-      case ISD::FEXP2:
-      case ISD::FCEIL:
-      case ISD::FFLOOR:
-      case ISD::FNEARBYINT:
-      case ISD::FRINT:
-      case ISD::FROUNDEVEN:
-      case ISD::FTRUNC:
-        IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 78;
-        break;
+    switch (Op) {
+    // Several BF16 instructions are available on sm_90 only.
+    case ISD::FADD:
+    case ISD::FMUL:
+    case ISD::FSUB:
+    case ISD::SELECT:
+    case ISD::SELECT_CC:
+    case ISD::SETCC:
+    case ISD::FEXP2:
+    case ISD::FCEIL:
+    case ISD::FFLOOR:
+    case ISD::FNEARBYINT:
+    case ISD::FRINT:
+    case ISD::FROUNDEVEN:
+    case ISD::FTRUNC:
+      IsOpSupported = STI.getSmVersion() >= 90 && STI.getPTXVersion() >= 78;
+      break;
+    // Several BF16 instructions are available on sm_80 only.
+    case ISD::FMINNUM:
+    case ISD::FMAXNUM:
+    case ISD::FMAXNUM_IEEE:
+    case ISD::FMINNUM_IEEE:
+    case ISD::FMAXIMUM:
+    case ISD::FMINIMUM:
+      IsOpSupported &= STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
+      break;
     }
     setOperationAction(
         Op, VT, IsOpSupported ? Action : NoBF16Action);
@@ -838,26 +859,23 @@ NVPTXTargetLowering::NVPTXTargetLowering(const NVPTXTargetMachine &TM,
       AddPromotedToType(Op, MVT::bf16, MVT::f32);
   }
 
-  // max.f16, max.f16x2 and max.NaN are supported on sm_80+.
-  auto GetMinMaxAction = [&](LegalizeAction NotSm80Action) {
-    bool IsAtLeastSm80 = STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
-    return IsAtLeastSm80 ? Legal : NotSm80Action;
-  };
   for (const auto &Op : {ISD::FMINNUM, ISD::FMAXNUM}) {
-    setFP16OperationAction(Op, MVT::f16, GetMinMaxAction(Promote), Promote);
     setOperationAction(Op, MVT::f32, Legal);
     setOperationAction(Op, MVT::f64, Legal);
-    setFP16OperationAction(Op, MVT::v2f16, GetMinMaxAction(Expand), Expand);
+    setFP16OperationAction(Op, MVT::f16, Legal, Promote);
+    setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
     setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
     setBF16OperationAction(Op, MVT::bf16, Legal, Promote);
     if (getOperationAction(Op, MVT::bf16) == Promote)
       AddPromotedToType(Op, MVT::bf16, MVT::f32);
   }
+  bool SupportsF32MinMaxNaN =
+      STI.getSmVersion() >= 80 && STI.getPTXVersion() >= 70;
   for (const auto &Op : {ISD::FMINIMUM, ISD::FMAXIMUM}) {
-    setFP16OperationAction(Op, MVT::f16, GetMinMaxAction(Expand), Expand);
-    setFP16OperationAction(Op, MVT::bf16, Legal, Expand);
-    setOperationAction(Op, MVT::f32, GetMinMaxAction(Expand));
-    setFP16OperationAction(Op, MVT::v2f16, GetMinMaxAction(Expand), Expand);
+    setOperationAction(Op, MVT::f32, SupportsF32MinMaxNaN ? Legal : Expand);
+    setFP16OperationAction(Op, MVT::f16, Legal, Expand);
+    setFP16OperationAction(Op, MVT::v2f16, Legal, Expand);
+    setBF16OperationAction(Op, MVT::bf16, Legal, Expand);
     setBF16OperationAction(Op, MVT::v2bf16, Legal, Expand);
   }
 

diff  --git a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
index 4cbab8bf24166c..f029e5f322968b 100644
--- a/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
+++ b/llvm/test/CodeGen/NVPTX/bf16-instructions.ll
@@ -1,344 +1,1455 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc < %s -march=nvptx64 -mcpu=sm_70 -mattr=+ptx71 | FileCheck --check-prefixes=CHECK,SM70 %s
 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | FileCheck --check-prefixes=CHECK,SM80 %s
 ; RUN: llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | FileCheck --check-prefixes=CHECK,SM90 %s
 ; RUN: %if ptxas-11.8 %{ llc < %s -march=nvptx64 -mcpu=sm_80 -mattr=+ptx71 | %ptxas-verify -arch=sm_80 %}
 ; RUN: %if ptxas-11.8 %{ llc < %s -march=nvptx64 -mcpu=sm_90 -mattr=+ptx78 | %ptxas-verify -arch=sm_90 %}
 
+target triple = "nvptx64-nvidia-cuda"
+
 ; LDST: .b8 bfloat_array[8] = {1, 2, 3, 4, 5, 6, 7, 8};
 @"bfloat_array" = addrspace(1) constant [4 x bfloat]
                 [bfloat 0xR0201, bfloat 0xR0403, bfloat 0xR0605, bfloat 0xR0807]
 
-; CHECK-LABEL: test_fadd(
-; CHECK-DAG:  ld.param.b16    [[A:%rs[0-9]+]], [test_fadd_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%rs[0-9]+]], [test_fadd_param_1];
-; SM90:       add.rn.bf16     [[R:%rs[0-9]+]], [[A]], [[B]];
-;
-; SM80-DAG:   cvt.f32.bf16    [[FA:%f[0-9]+]], [[A]];
-; SM80-DAG:   cvt.f32.bf16    [[FB:%f[0-9]+]], [[B]];
-; SM80:       add.rn.f32      [[FR:%f[0-9]+]], [[FA]], [[FB]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[FR]];
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-
 define bfloat @test_fadd(bfloat %0, bfloat %1) {
+; SM70-LABEL: test_fadd(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<11>;
+; SM70-NEXT:    .reg .f32 %f<4>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fadd_param_1];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    ld.param.u16 %r3, [test_fadd_param_0];
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f2, %r4;
+; SM70-NEXT:    add.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r5, %f3;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r10; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fadd(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-NEXT:    .reg .f32 %f<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fadd_param_0];
+; SM80-NEXT:    ld.param.b16 %rs2, [test_fadd_param_1];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs1;
+; SM80-NEXT:    add.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f3;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fadd(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fadd_param_0];
+; SM90-NEXT:    ld.param.b16 %rs2, [test_fadd_param_1];
+; SM90-NEXT:    add.rn.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
   %3 = fadd bfloat %0, %1
   ret bfloat %3
 }
 
-; CHECK-LABEL: test_fsub(
-; CHECK-DAG:  ld.param.b16    [[A:%rs[0-9]+]], [test_fsub_param_0];
-; CHECK-DAG:  ld.param.b16    [[B:%rs[0-9]+]], [test_fsub_param_1];
-; SM90:       sub.rn.bf16     [[R:%rs[0-9]+]], [[A]], [[B]];
-;
-; SM80-DAG:   cvt.f32.bf16    [[FA:%f[0-9]+]], [[A]];
-; SM80-DAG:   cvt.f32.bf16    [[FB:%f[0-9]+]], [[B]];
-; SM80:       sub.rn.f32      [[FR:%f[0-9]+]], [[FA]], [[FB]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[FR]];
-; CHECK-NEXT: st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-
 define bfloat @test_fsub(bfloat %0, bfloat %1) {
+; SM70-LABEL: test_fsub(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<11>;
+; SM70-NEXT:    .reg .f32 %f<4>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fsub_param_1];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    ld.param.u16 %r3, [test_fsub_param_0];
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f2, %r4;
+; SM70-NEXT:    sub.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r5, %f3;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r10; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fsub(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-NEXT:    .reg .f32 %f<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fsub_param_0];
+; SM80-NEXT:    ld.param.b16 %rs2, [test_fsub_param_1];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs1;
+; SM80-NEXT:    sub.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f3;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fsub(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fsub_param_0];
+; SM90-NEXT:    ld.param.b16 %rs2, [test_fsub_param_1];
+; SM90-NEXT:    sub.rn.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
   %3 = fsub bfloat %0, %1
   ret bfloat %3
 }
 
-; CHECK-LABEL: test_faddx2(
-; CHECK-DAG:  ld.param.b32    [[A:%r[0-9]+]], [test_faddx2_param_0];
-; CHECK-DAG:  ld.param.b32    [[B:%r[0-9]+]], [test_faddx2_param_1];
-; SM90:       add.rn.bf16x2   [[R:%r[0-9]+]], [[A]], [[B]];
-
-; SM80-DAG:   mov.b32         {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]];
-; SM80-DAG:   mov.b32         {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]];
-; SM80-DAG:   cvt.f32.bf16    [[FA1:%f[0-9]+]], [[A1]];
-; SM80-DAG:   cvt.f32.bf16    [[FA0:%f[0-9]+]], [[A0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB0:%f[0-9]+]], [[B0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB1:%f[0-9]+]], [[B1]];
-; SM80-DAG:   add.rn.f32      [[FR0:%f[0-9]+]], [[FA0]], [[FB0]];
-; SM80-DAG:   add.rn.f32      [[FR1:%f[0-9]+]], [[FA1]], [[FB1]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]];
-; SM80:       mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]};
-; CHECK:      st.param.b32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-
 define <2 x bfloat> @test_faddx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; SM70-LABEL: test_faddx2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_faddx2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_faddx2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    add.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    add.rn.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_faddx2(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-NEXT:    .reg .f32 %f<7>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_faddx2_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_faddx2_param_1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    add.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    add.rn.f32 %f6, %f5, %f4;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_faddx2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_faddx2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_faddx2_param_0];
+; SM90-NEXT:    add.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
   %r = fadd <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
 }
 
-; CHECK-LABEL: test_fsubx2(
-; CHECK-DAG:  ld.param.b32    [[A:%r[0-9]+]], [test_fsubx2_param_0];
-; CHECK-DAG:  ld.param.b32    [[B:%r[0-9]+]], [test_fsubx2_param_1];
-; SM90:       sub.rn.bf16x2   [[R:%r[0-9]+]], [[A]], [[B]];
-
-; SM80-DAG:   mov.b32         {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]];
-; SM80-DAG:   mov.b32         {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]];
-; SM80-DAG:   cvt.f32.bf16    [[FA1:%f[0-9]+]], [[A1]];
-; SM80-DAG:   cvt.f32.bf16    [[FA0:%f[0-9]+]], [[A0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB0:%f[0-9]+]], [[B0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB1:%f[0-9]+]], [[B1]];
-; SM80-DAG:   sub.rn.f32      [[FR0:%f[0-9]+]], [[FA0]], [[FB0]];
-; SM80-DAG:   sub.rn.f32      [[FR1:%f[0-9]+]], [[FA1]], [[FB1]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]];
-; SM80:       mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]};
-
-; CHECK:      st.param.b32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-
 define <2 x bfloat> @test_fsubx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; SM70-LABEL: test_fsubx2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    sub.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    sub.rn.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fsubx2(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-NEXT:    .reg .f32 %f<7>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    sub.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    sub.rn.f32 %f6, %f5, %f4;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fsubx2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_fsubx2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fsubx2_param_0];
+; SM90-NEXT:    sub.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
   %r = fsub <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
 }
 
-; CHECK-LABEL: test_fmulx2(
-; CHECK-DAG:  ld.param.b32    [[A:%r[0-9]+]], [test_fmulx2_param_0];
-; CHECK-DAG:  ld.param.b32    [[B:%r[0-9]+]], [test_fmulx2_param_1];
-; SM90:       mul.rn.bf16x2   [[R:%r[0-9]+]], [[A]], [[B]];
-
-; SM80-DAG:   mov.b32         {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]];
-; SM80-DAG:   mov.b32         {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]];
-; SM80-DAG:   cvt.f32.bf16    [[FA1:%f[0-9]+]], [[A1]];
-; SM80-DAG:   cvt.f32.bf16    [[FA0:%f[0-9]+]], [[A0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB0:%f[0-9]+]], [[B0]];
-; SM80-DAG:   cvt.f32.bf16    [[FB1:%f[0-9]+]], [[B1]];
-; SM80-DAG:   mul.rn.f32      [[FR0:%f[0-9]+]], [[FA0]], [[FB0]];
-; SM80-DAG:   mul.rn.f32      [[FR1:%f[0-9]+]], [[FA1]], [[FB1]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R0:%rs[0-9]+]], [[FR0]];
-; SM80-DAG:   cvt.rn.bf16.f32 [[R1:%rs[0-9]+]], [[FR1]];
-; SM80:       mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]};
-
-; CHECK:      st.param.b32    [func_retval0+0], [[R]];
-; CHECK:      ret;
-
 define <2 x bfloat> @test_fmulx2(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; SM70-LABEL: test_fmulx2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    mul.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    mul.rn.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fmulx2(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-NEXT:    .reg .f32 %f<7>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    mul.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    mul.rn.f32 %f6, %f5, %f4;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fmulx2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_fmulx2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fmulx2_param_0];
+; SM90-NEXT:    mul.rn.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
   %r = fmul <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
 }
 
-; CHECK-LABEL: test_fdiv(
-; CHECK-DAG:  ld.param.b32    [[A:%r[0-9]+]], [test_fdiv_param_0];
-; CHECK-DAG:  ld.param.b32    [[B:%r[0-9]+]], [test_fdiv_param_1];
-; CHECK-DAG:  mov.b32         {[[A0:%rs[0-9]+]], [[A1:%rs[0-9]+]]}, [[A]]
-; CHECK-DAG:  mov.b32         {[[B0:%rs[0-9]+]], [[B1:%rs[0-9]+]]}, [[B]]
-; CHECK-DAG:  cvt.f32.bf16     [[FA0:%f[0-9]+]], [[A0]];
-; CHECK-DAG:  cvt.f32.bf16     [[FA1:%f[0-9]+]], [[A1]];
-; CHECK-DAG:  cvt.f32.bf16     [[FB0:%f[0-9]+]], [[B0]];
-; CHECK-DAG:  cvt.f32.bf16     [[FB1:%f[0-9]+]], [[B1]];
-; CHECK-DAG:  div.rn.f32      [[FR0:%f[0-9]+]], [[FA0]], [[FB0]];
-; CHECK-DAG:  div.rn.f32      [[FR1:%f[0-9]+]], [[FA1]], [[FB1]];
-; CHECK-DAG:  cvt.rn.bf16.f32  [[R0:%rs[0-9]+]], [[FR0]];
-; CHECK-DAG:  cvt.rn.bf16.f32  [[R1:%rs[0-9]+]], [[FR1]];
-; CHECK-NEXT: mov.b32         [[R:%r[0-9]+]], {[[R0]], [[R1]]}
-; CHECK-NEXT: st.param.b32    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
-
 define <2 x bfloat> @test_fdiv(<2 x bfloat> %a, <2 x bfloat> %b) #0 {
+; SM70-LABEL: test_fdiv(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    div.rn.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    div.rn.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fdiv(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<7>;
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-NEXT:    .reg .f32 %f<7>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
+; SM80-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM80-NEXT:    div.rn.f32 %f3, %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM80-NEXT:    div.rn.f32 %f6, %f5, %f4;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM80-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fdiv(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<7>;
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-NEXT:    .reg .f32 %f<7>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_fdiv_param_0];
+; SM90-NEXT:    ld.param.b32 %r2, [test_fdiv_param_1];
+; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM90-NEXT:    cvt.f32.bf16 %f1, %rs2;
+; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r1;
+; SM90-NEXT:    cvt.f32.bf16 %f2, %rs4;
+; SM90-NEXT:    div.rn.f32 %f3, %f2, %f1;
+; SM90-NEXT:    cvt.rn.bf16.f32 %rs5, %f3;
+; SM90-NEXT:    cvt.f32.bf16 %f4, %rs1;
+; SM90-NEXT:    cvt.f32.bf16 %f5, %rs3;
+; SM90-NEXT:    div.rn.f32 %f6, %f5, %f4;
+; SM90-NEXT:    cvt.rn.bf16.f32 %rs6, %f6;
+; SM90-NEXT:    mov.b32 %r3, {%rs6, %rs5};
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
   %r = fdiv <2 x bfloat> %a, %b
   ret <2 x bfloat> %r
 }
 
-; CHECK-LABEL: test_extract_0(
-; CHECK:      ld.param.b16    [[A:%rs[0-9]+]], [test_extract_0_param_0];
-; CHECK:      st.param.b16    [func_retval0+0], [[A]];
-; CHECK:      ret;
-
 define bfloat @test_extract_0(<2 x bfloat> %a) #0 {
+; CHECK-LABEL: test_extract_0(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [test_extract_0_param_0];
+; CHECK-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; CHECK-NEXT:    ret;
   %e = extractelement <2 x bfloat> %a, i32 0
   ret bfloat %e
 }
 
-; CHECK-LABEL: test_extract_1(
-; CHECK:      ld.param.b16    [[A:%rs[0-9]+]], [test_extract_1_param_0+2];
-; CHECK:      st.param.b16    [func_retval0+0], [[A]];
-; CHECK:      ret;
-
 define bfloat @test_extract_1(<2 x bfloat> %a) #0 {
+; CHECK-LABEL: test_extract_1(
+; CHECK:       {
+; CHECK-NEXT:    .reg .b16 %rs<2>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.b16 %rs1, [test_extract_1_param_0+2];
+; CHECK-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; CHECK-NEXT:    ret;
   %e = extractelement <2 x bfloat> %a, i32 1
   ret bfloat %e
 }
 
-; CHECK-LABEL: test_fpext_float(
-; CHECK:      ld.param.b16    [[A:%rs[0-9]+]], [test_fpext_float_param_0];
-; CHECK:      cvt.f32.bf16     [[R:%f[0-9]+]], [[A]];
-; CHECK:      st.param.f32    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define float @test_fpext_float(bfloat %a) #0 {
+; SM70-LABEL: test_fpext_float(
+; SM70:       {
+; SM70-NEXT:    .reg .b32 %r<3>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fpext_float_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    st.param.f32 [func_retval0+0], %f1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fpext_float(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fpext_float_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    st.param.f32 [func_retval0+0], %f1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fpext_float(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<2>;
+; SM90-NEXT:    .reg .f32 %f<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fpext_float_param_0];
+; SM90-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM90-NEXT:    st.param.f32 [func_retval0+0], %f1;
+; SM90-NEXT:    ret;
   %r = fpext bfloat %a to float
   ret float %r
 }
 
-; CHECK-LABEL: test_fptrunc_float(
-; CHECK:      ld.param.f32    [[A:%f[0-9]+]], [test_fptrunc_float_param_0];
-; CHECK:      cvt.rn.bf16.f32  [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_fptrunc_float(float %a) #0 {
+; SM70-LABEL: test_fptrunc_float(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.f32 %f1, [test_fptrunc_float_param_0];
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fptrunc_float(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.f32 %f1, [test_fptrunc_float_param_0];
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fptrunc_float(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<2>;
+; SM90-NEXT:    .reg .f32 %f<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.f32 %f1, [test_fptrunc_float_param_0];
+; SM90-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM90-NEXT:    ret;
   %r = fptrunc float %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_fadd_imm_1(
-; CHECK:      ld.param.b16    [[A:%rs[0-9]+]], [test_fadd_imm_1_param_0];
-; SM90:       mov.b16         [[B:%rs[0-9]+]], 0x3F80;
-; SM90:       add.rn.bf16     [[R:%rs[0-9]+]], [[A]], [[B]];
-
-; SM80-DAG:   cvt.f32.bf16    [[FA:%f[0-9]+]], [[A]];
-; SM80:       add.rn.f32      [[FR:%f[0-9]+]], [[FA]], 0f3F800000;
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[FR]];
-
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT: ret;
 define bfloat @test_fadd_imm_1(bfloat %a) #0 {
+; SM70-LABEL: test_fadd_imm_1(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<9>;
+; SM70-NEXT:    .reg .f32 %f<3>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fadd_imm_1_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    add.rn.f32 %f2, %f1, 0f3F800000;
+; SM70-NEXT:    mov.b32 %r3, %f2;
+; SM70-NEXT:    bfe.u32 %r4, %r3, 16, 1;
+; SM70-NEXT:    add.s32 %r5, %r4, %r3;
+; SM70-NEXT:    add.s32 %r6, %r5, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f2, %f2;
+; SM70-NEXT:    or.b32 %r7, %r3, 4194304;
+; SM70-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r8; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fadd_imm_1(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<3>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fadd_imm_1_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    add.rn.f32 %f2, %f1, 0f3F800000;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f2;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fadd_imm_1(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fadd_imm_1_param_0];
+; SM90-NEXT:    mov.b16 %rs2, 0x3F80;
+; SM90-NEXT:    add.rn.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
   %r = fadd bfloat %a, 1.0
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_select_cc_bf16_f64(
-; CHECK-DAG:      ld.param.f64    [[A:%fd[0-9]+]], [test_select_cc_bf16_f64_param_0];
-; CHECK-DAG:      ld.param.f64    [[B:%fd[0-9]+]], [test_select_cc_bf16_f64_param_1];
-; CHECK:          setp.lt.f64     [[P:%p[0-9]+]], [[A]], [[B]];
-; CHECK-DAG:      ld.param.b16    [[C:%rs[0-9]+]], [test_select_cc_bf16_f64_param_2];
-; CHECK-DAG:      ld.param.b16    [[D:%rs[0-9]+]], [test_select_cc_bf16_f64_param_3];
-; CHECK:          selp.b16        [[R:%rs[0-9]+]], [[C]], [[D]], [[P]];
-; CHECK-NEXT:     st.param.b16    [func_retval0+0], [[R]];
-; CHECK-NEXT:     ret;
 define bfloat @test_select_cc_bf16_f64(double %a, double %b, bfloat %c, bfloat %d) #0 {
+; CHECK-LABEL: test_select_cc_bf16_f64(
+; CHECK:       {
+; CHECK-NEXT:    .reg .pred %p<2>;
+; CHECK-NEXT:    .reg .b16 %rs<4>;
+; CHECK-NEXT:    .reg .f64 %fd<3>;
+; CHECK-EMPTY:
+; CHECK-NEXT:  // %bb.0:
+; CHECK-NEXT:    ld.param.f64 %fd1, [test_select_cc_bf16_f64_param_0];
+; CHECK-NEXT:    ld.param.f64 %fd2, [test_select_cc_bf16_f64_param_1];
+; CHECK-NEXT:    setp.lt.f64 %p1, %fd1, %fd2;
+; CHECK-NEXT:    ld.param.b16 %rs1, [test_select_cc_bf16_f64_param_2];
+; CHECK-NEXT:    ld.param.b16 %rs2, [test_select_cc_bf16_f64_param_3];
+; CHECK-NEXT:    selp.b16 %rs3, %rs1, %rs2, %p1;
+; CHECK-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; CHECK-NEXT:    ret;
   %cc = fcmp olt double %a, %b
   %r = select i1 %cc, bfloat %c, bfloat %d
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_extload_bf16x8
-; CHECK: ld.shared.v4.b32 {%r
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; CHECK: mov.b32 {%rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
-; SM80: cvt.f32.bf16 %f{{.*}}, %rs
 define <8 x float> @test_extload_bf16x8(ptr addrspace(3) noundef %arg) #0 {
+; SM70-LABEL: test_extload_bf16x8(
+; SM70:       {
+; SM70-NEXT:    .reg .b16 %rs<17>;
+; SM70-NEXT:    .reg .b32 %r<21>;
+; SM70-NEXT:    .reg .f32 %f<9>;
+; SM70-NEXT:    .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u64 %rd1, [test_extload_bf16x8_param_0];
+; SM70-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM70-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; SM70-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
+; SM70-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs8;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f1, %r6;
+; SM70-NEXT:    cvt.u32.u16 %r7, %rs7;
+; SM70-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NEXT:    mov.b32 %f2, %r8;
+; SM70-NEXT:    cvt.u32.u16 %r9, %rs6;
+; SM70-NEXT:    shl.b32 %r10, %r9, 16;
+; SM70-NEXT:    mov.b32 %f3, %r10;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs5;
+; SM70-NEXT:    shl.b32 %r12, %r11, 16;
+; SM70-NEXT:    mov.b32 %f4, %r12;
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs4;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f5, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs3;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f6, %r16;
+; SM70-NEXT:    cvt.u32.u16 %r17, %rs2;
+; SM70-NEXT:    shl.b32 %r18, %r17, 16;
+; SM70-NEXT:    mov.b32 %f7, %r18;
+; SM70-NEXT:    cvt.u32.u16 %r19, %rs1;
+; SM70-NEXT:    shl.b32 %r20, %r19, 16;
+; SM70-NEXT:    mov.b32 %f8, %r20;
+; SM70-NEXT:    st.param.v4.f32 [func_retval0+0], {%f8, %f7, %f6, %f5};
+; SM70-NEXT:    st.param.v4.f32 [func_retval0+16], {%f4, %f3, %f2, %f1};
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_extload_bf16x8(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<9>;
+; SM80-NEXT:    .reg .b32 %r<5>;
+; SM80-NEXT:    .reg .f32 %f<9>;
+; SM80-NEXT:    .reg .b64 %rd<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u64 %rd1, [test_extload_bf16x8_param_0];
+; SM80-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM80-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM80-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; SM80-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
+; SM80-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs8;
+; SM80-NEXT:    cvt.f32.bf16 %f2, %rs7;
+; SM80-NEXT:    cvt.f32.bf16 %f3, %rs6;
+; SM80-NEXT:    cvt.f32.bf16 %f4, %rs5;
+; SM80-NEXT:    cvt.f32.bf16 %f5, %rs4;
+; SM80-NEXT:    cvt.f32.bf16 %f6, %rs3;
+; SM80-NEXT:    cvt.f32.bf16 %f7, %rs2;
+; SM80-NEXT:    cvt.f32.bf16 %f8, %rs1;
+; SM80-NEXT:    st.param.v4.f32 [func_retval0+0], {%f8, %f7, %f6, %f5};
+; SM80-NEXT:    st.param.v4.f32 [func_retval0+16], {%f4, %f3, %f2, %f1};
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_extload_bf16x8(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<9>;
+; SM90-NEXT:    .reg .b32 %r<5>;
+; SM90-NEXT:    .reg .f32 %f<9>;
+; SM90-NEXT:    .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u64 %rd1, [test_extload_bf16x8_param_0];
+; SM90-NEXT:    ld.shared.v4.b32 {%r1, %r2, %r3, %r4}, [%rd1];
+; SM90-NEXT:    mov.b32 {%rs1, %rs2}, %r1;
+; SM90-NEXT:    mov.b32 {%rs3, %rs4}, %r2;
+; SM90-NEXT:    mov.b32 {%rs5, %rs6}, %r3;
+; SM90-NEXT:    mov.b32 {%rs7, %rs8}, %r4;
+; SM90-NEXT:    cvt.f32.bf16 %f1, %rs8;
+; SM90-NEXT:    cvt.f32.bf16 %f2, %rs7;
+; SM90-NEXT:    cvt.f32.bf16 %f3, %rs6;
+; SM90-NEXT:    cvt.f32.bf16 %f4, %rs5;
+; SM90-NEXT:    cvt.f32.bf16 %f5, %rs4;
+; SM90-NEXT:    cvt.f32.bf16 %f6, %rs3;
+; SM90-NEXT:    cvt.f32.bf16 %f7, %rs2;
+; SM90-NEXT:    cvt.f32.bf16 %f8, %rs1;
+; SM90-NEXT:    st.param.v4.f32 [func_retval0+0], {%f8, %f7, %f6, %f5};
+; SM90-NEXT:    st.param.v4.f32 [func_retval0+16], {%f4, %f3, %f2, %f1};
+; SM90-NEXT:    ret;
   %load = load <8 x bfloat>, ptr addrspace(3) %arg, align 16
   %res = fpext <8 x bfloat> %load to <8 x float>
   ret <8 x float> %res
 }
 
-; CHECK-LABEL: test_fptosi_i16(
-; CHECK:      ld.param.b16     [[A:%rs[0-9]+]], [test_fptosi_i16_param_0];
-; SM80:       cvt.f32.bf16     [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rzi.s16.f32  [[C:%rs[0-9]+]], [[B]];
-; SM80:       cvt.u32.u16      [[R:%r[0-9]+]], [[C]];
-; SM90:       cvt.rzi.s16.bf16 [[B:%rs[0-9]+]], [[A]];
-; SM90:       cvt.u32.u16      [[R:%r[0-9]+]], [[B]];
-; CHECK:      st.param.b32     [func_retval0+0], [[R]];
-; CHECK:      ret;
 define i16 @test_fptosi_i16(bfloat %a) {
+; SM70-LABEL: test_fptosi_i16(
+; SM70:       {
+; SM70-NEXT:    .reg .b16 %rs<2>;
+; SM70-NEXT:    .reg .b32 %r<4>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fptosi_i16_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    cvt.rzi.s16.f32 %rs1, %f1;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs1;
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fptosi_i16(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .b32 %r<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fptosi_i16_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    cvt.rzi.s16.f32 %rs2, %f1;
+; SM80-NEXT:    cvt.u32.u16 %r1, %rs2;
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fptosi_i16(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-NEXT:    .reg .b32 %r<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fptosi_i16_param_0];
+; SM90-NEXT:    cvt.rzi.s16.bf16 %rs2, %rs1;
+; SM90-NEXT:    cvt.u32.u16 %r1, %rs2;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; SM90-NEXT:    ret;
   %r = fptosi bfloat %a to i16
   ret i16 %r
 }
 
-; CHECK-LABEL: test_fptoui_i16(
-; CHECK:      ld.param.b16     [[A:%rs[0-9]+]], [test_fptoui_i16_param_0];
-; SM80:       cvt.f32.bf16     [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rzi.u16.f32  [[C:%rs[0-9]+]], [[B]];
-; SM80:       cvt.u32.u16      [[R:%r[0-9]+]], [[C]];
-; SM90:       cvt.rzi.u16.bf16 [[B:%rs[0-9]+]], [[A]];
-; SM90:       cvt.u32.u16      [[R:%r[0-9]+]], [[B]];
-; CHECK:      st.param.b32     [func_retval0+0], [[R]];
-; CHECK:      ret;
 define i16 @test_fptoui_i16(bfloat %a) {
+; SM70-LABEL: test_fptoui_i16(
+; SM70:       {
+; SM70-NEXT:    .reg .b16 %rs<2>;
+; SM70-NEXT:    .reg .b32 %r<4>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_fptoui_i16_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    cvt.rzi.u16.f32 %rs1, %f1;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs1;
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_fptoui_i16(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .b32 %r<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_fptoui_i16_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    cvt.rzi.u16.f32 %rs2, %f1;
+; SM80-NEXT:    cvt.u32.u16 %r1, %rs2;
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_fptoui_i16(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-NEXT:    .reg .b32 %r<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_fptoui_i16_param_0];
+; SM90-NEXT:    cvt.rzi.u16.bf16 %rs2, %rs1;
+; SM90-NEXT:    cvt.u32.u16 %r1, %rs2;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r1;
+; SM90-NEXT:    ret;
   %r = fptoui bfloat %a to i16
   ret i16 %r
 }
 
-; CHECK-LABEL: test_sitofp_i16(
-; CHECK:      ld.param.u16    [[A:%rs[0-9]+]], [test_sitofp_i16_param_0];
-; SM80:       cvt.rn.f32.s16  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.s16 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_sitofp_i16(i16 %a) {
+; SM70-LABEL: test_sitofp_i16(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<4>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %rs1, [test_sitofp_i16_param_0];
+; SM70-NEXT:    cvt.rn.f32.s16 %f1, %rs1;
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_sitofp_i16(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u16 %rs1, [test_sitofp_i16_param_0];
+; SM80-NEXT:    cvt.rn.f32.s16 %f1, %rs1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_sitofp_i16(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u16 %rs1, [test_sitofp_i16_param_0];
+; SM90-NEXT:    cvt.rn.bf16.s16 %rs2, %rs1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM90-NEXT:    ret;
   %r = sitofp i16 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i8(
-; CHECK:      ld.param.u8 %rs1, [test_uitofp_i8_param_0];
-; SM80:       cvt.rn.f32.u16  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.u16 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i8(i8 %a) {
+; SM70-LABEL: test_uitofp_i8(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<4>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u8 %rs1, [test_uitofp_i8_param_0];
+; SM70-NEXT:    cvt.rn.f32.u16 %f1, %rs1;
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i8(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u8 %rs1, [test_uitofp_i8_param_0];
+; SM80-NEXT:    cvt.rn.f32.u16 %f1, %rs1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i8(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u8 %rs1, [test_uitofp_i8_param_0];
+; SM90-NEXT:    cvt.rn.bf16.u16 %rs2, %rs1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM90-NEXT:    ret;
   %r = uitofp i8 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i1(
-; CHECK:      ld.param.u8     [[A:%rs[0-9]+]], [test_uitofp_i1_param_0];
-; CHECK:      and.b16         [[B:%rs[0-9]+]], [[A]], 1;
-; CHECK:      setp.eq.b16     [[C:%p[0-9]+]], [[B]], 1;
-; CHECK:      selp.u32        [[D:%r[0-9]+]], 1, 0, [[C]];
-; SM80:       cvt.rn.f32.u32  [[E:%f[0-9]+]], [[D]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[E]];
-; SM90:       cvt.rn.bf16.u32 [[R:%rs[0-9]+]], [[D]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i1(i1 %a) {
+; SM70-LABEL: test_uitofp_i1(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<5>;
+; SM70-NEXT:    .reg .b32 %r<8>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u8 %rs1, [test_uitofp_i1_param_0];
+; SM70-NEXT:    and.b16 %rs2, %rs1, 1;
+; SM70-NEXT:    setp.eq.b16 %p1, %rs2, 1;
+; SM70-NEXT:    selp.u32 %r1, 1, 0, %p1;
+; SM70-NEXT:    cvt.rn.f32.u32 %f1, %r1;
+; SM70-NEXT:    mov.b32 %r2, %f1;
+; SM70-NEXT:    bfe.u32 %r3, %r2, 16, 1;
+; SM70-NEXT:    add.s32 %r4, %r3, %r2;
+; SM70-NEXT:    add.s32 %r5, %r4, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f1, %f1;
+; SM70-NEXT:    or.b32 %r6, %r2, 4194304;
+; SM70-NEXT:    selp.b32 %r7, %r6, %r5, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs3}, %r7; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i1(
+; SM80:       {
+; SM80-NEXT:    .reg .pred %p<2>;
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-NEXT:    .reg .b32 %r<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u8 %rs1, [test_uitofp_i1_param_0];
+; SM80-NEXT:    and.b16 %rs2, %rs1, 1;
+; SM80-NEXT:    setp.eq.b16 %p1, %rs2, 1;
+; SM80-NEXT:    selp.u32 %r1, 1, 0, %p1;
+; SM80-NEXT:    cvt.rn.f32.u32 %f1, %r1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs3, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i1(
+; SM90:       {
+; SM90-NEXT:    .reg .pred %p<2>;
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-NEXT:    .reg .b32 %r<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u8 %rs1, [test_uitofp_i1_param_0];
+; SM90-NEXT:    and.b16 %rs2, %rs1, 1;
+; SM90-NEXT:    setp.eq.b16 %p1, %rs2, 1;
+; SM90-NEXT:    selp.u32 %r1, 1, 0, %p1;
+; SM90-NEXT:    cvt.rn.bf16.u32 %rs3, %r1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
   %r = uitofp i1 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i16(
-; CHECK:      ld.param.u16    [[A:%rs[0-9]+]], [test_uitofp_i16_param_0];
-; SM80:       cvt.rn.f32.u16  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.u16 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i16(i16 %a) {
+; SM70-LABEL: test_uitofp_i16(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<4>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %rs1, [test_uitofp_i16_param_0];
+; SM70-NEXT:    cvt.rn.f32.u16 %f1, %rs1;
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs2}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i16(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u16 %rs1, [test_uitofp_i16_param_0];
+; SM80-NEXT:    cvt.rn.f32.u16 %f1, %rs1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i16(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u16 %rs1, [test_uitofp_i16_param_0];
+; SM90-NEXT:    cvt.rn.bf16.u16 %rs2, %rs1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM90-NEXT:    ret;
   %r = uitofp i16 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i32(
-; CHECK:      ld.param.u32    [[A:%r[0-9]+]], [test_uitofp_i32_param_0];
-; SM80:       cvt.rn.f32.u32  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.u32 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i32(i32 %a) {
+; SM70-LABEL: test_uitofp_i32(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<8>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u32 %r1, [test_uitofp_i32_param_0];
+; SM70-NEXT:    cvt.rn.f32.u32 %f1, %r1;
+; SM70-NEXT:    mov.b32 %r2, %f1;
+; SM70-NEXT:    bfe.u32 %r3, %r2, 16, 1;
+; SM70-NEXT:    add.s32 %r4, %r3, %r2;
+; SM70-NEXT:    add.s32 %r5, %r4, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r6, %r2, 4194304;
+; SM70-NEXT:    selp.b32 %r7, %r6, %r5, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r7; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i32(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<2>;
+; SM80-NEXT:    .reg .b32 %r<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u32 %r1, [test_uitofp_i32_param_0];
+; SM80-NEXT:    cvt.rn.f32.u32 %f1, %r1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i32(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<2>;
+; SM90-NEXT:    .reg .b32 %r<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u32 %r1, [test_uitofp_i32_param_0];
+; SM90-NEXT:    cvt.rn.bf16.u32 %rs1, %r1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM90-NEXT:    ret;
   %r = uitofp i32 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_uitofp_i64(
-; CHECK:      ld.param.u64    [[A:%rd[0-9]+]], [test_uitofp_i64_param_0];
-; SM80:       cvt.rn.f32.u64  [[B:%f[0-9]+]], [[A]];
-; SM80:       cvt.rn.bf16.f32 [[R:%rs[0-9]+]], [[B]];
-; SM90:       cvt.rn.bf16.u64 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16    [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_uitofp_i64(i64 %a) {
+; SM70-LABEL: test_uitofp_i64(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<2>;
+; SM70-NEXT:    .reg .b64 %rd<2>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u64 %rd1, [test_uitofp_i64_param_0];
+; SM70-NEXT:    cvt.rn.f32.u64 %f1, %rd1;
+; SM70-NEXT:    mov.b32 %r1, %f1;
+; SM70-NEXT:    bfe.u32 %r2, %r1, 16, 1;
+; SM70-NEXT:    add.s32 %r3, %r2, %r1;
+; SM70-NEXT:    add.s32 %r4, %r3, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f1, %f1;
+; SM70-NEXT:    or.b32 %r5, %r1, 4194304;
+; SM70-NEXT:    selp.b32 %r6, %r5, %r4, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r6; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_uitofp_i64(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<2>;
+; SM80-NEXT:    .reg .f32 %f<2>;
+; SM80-NEXT:    .reg .b64 %rd<2>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.u64 %rd1, [test_uitofp_i64_param_0];
+; SM80-NEXT:    cvt.rn.f32.u64 %f1, %rd1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs1, %f1;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_uitofp_i64(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<2>;
+; SM90-NEXT:    .reg .b64 %rd<2>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.u64 %rd1, [test_uitofp_i64_param_0];
+; SM90-NEXT:    cvt.rn.bf16.u64 %rs1, %rd1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM90-NEXT:    ret;
   %r = uitofp i64 %a to bfloat
   ret bfloat %r
 }
 
-; CHECK-LABEL: test_roundeven(
-; CHECK:      ld.param.b16      [[A:%rs[0-9]+]], [test_roundeven_param_0];
-; SM80:       cvt.rni.f32.f32   [[F:%f[0-9]+]]
-; SM80:       cvt.rn.bf16.f32   [[R:%rs[0-9]+]], [[F]];
-; SM90:       cvt.rni.bf16.bf16 [[R:%rs[0-9]+]], [[A]];
-; CHECK:      st.param.b16      [func_retval0+0], [[R]];
-; CHECK:      ret;
 define bfloat @test_roundeven(bfloat %a) {
+; SM70-LABEL: test_roundeven(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<9>;
+; SM70-NEXT:    .reg .f32 %f<3>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_roundeven_param_0];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    cvt.rni.f32.f32 %f2, %f1;
+; SM70-NEXT:    mov.b32 %r3, %f2;
+; SM70-NEXT:    bfe.u32 %r4, %r3, 16, 1;
+; SM70-NEXT:    add.s32 %r5, %r4, %r3;
+; SM70-NEXT:    add.s32 %r6, %r5, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f2, %f2;
+; SM70-NEXT:    or.b32 %r7, %r3, 4194304;
+; SM70-NEXT:    selp.b32 %r8, %r7, %r6, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r8; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_roundeven(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<3>;
+; SM80-NEXT:    .reg .f32 %f<3>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_roundeven_param_0];
+; SM80-NEXT:    cvt.f32.bf16 %f1, %rs1;
+; SM80-NEXT:    cvt.rni.f32.f32 %f2, %f1;
+; SM80-NEXT:    cvt.rn.bf16.f32 %rs2, %f2;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_roundeven(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<3>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_roundeven_param_0];
+; SM90-NEXT:    cvt.rni.bf16.bf16 %rs2, %rs1;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs2;
+; SM90-NEXT:    ret;
   %r = call bfloat @llvm.roundeven.bf16(bfloat %a)
   ret bfloat %r
 }
+
+define bfloat @test_maximum(bfloat %a, bfloat %b) {
+; SM70-LABEL: test_maximum(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<6>;
+; SM70-NEXT:    .reg .b16 %rs<11>;
+; SM70-NEXT:    .reg .b32 %r<7>;
+; SM70-NEXT:    .reg .f32 %f<4>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b16 %rs1, [test_maximum_param_0];
+; SM70-NEXT:    ld.param.b16 %rs3, [test_maximum_param_1];
+; SM70-NEXT:    cvt.u32.u16 %r1, %rs3;
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs1;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f2, %r4;
+; SM70-NEXT:    setp.gt.f32 %p1, %f2, %f1;
+; SM70-NEXT:    selp.b16 %rs5, %rs1, %rs3, %p1;
+; SM70-NEXT:    setp.nan.f32 %p2, %f2, %f1;
+; SM70-NEXT:    selp.b16 %rs6, 0x7FC0, %rs5, %p2;
+; SM70-NEXT:    setp.eq.s16 %p3, %rs1, 0;
+; SM70-NEXT:    selp.b16 %rs7, %rs1, %rs6, %p3;
+; SM70-NEXT:    setp.eq.s16 %p4, %rs3, 0;
+; SM70-NEXT:    selp.b16 %rs8, %rs3, %rs7, %p4;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs6;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f3, %r6;
+; SM70-NEXT:    setp.eq.f32 %p5, %f3, 0f00000000;
+; SM70-NEXT:    selp.b16 %rs10, %rs8, %rs6, %p5;
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs10;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_maximum(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_maximum_param_0];
+; SM80-NEXT:    ld.param.b16 %rs2, [test_maximum_param_1];
+; SM80-NEXT:    max.NaN.bf16 %rs3, %rs1, %rs2;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_maximum(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_maximum_param_0];
+; SM90-NEXT:    ld.param.b16 %rs2, [test_maximum_param_1];
+; SM90-NEXT:    max.NaN.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
+  %r = call bfloat @llvm.maximum.bf16(bfloat %a, bfloat %b)
+  ret bfloat %r
+}
+
+define bfloat @test_maxnum(bfloat %a, bfloat %b) {
+; SM70-LABEL: test_maxnum(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<2>;
+; SM70-NEXT:    .reg .b16 %rs<3>;
+; SM70-NEXT:    .reg .b32 %r<11>;
+; SM70-NEXT:    .reg .f32 %f<4>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.u16 %r1, [test_maxnum_param_1];
+; SM70-NEXT:    shl.b32 %r2, %r1, 16;
+; SM70-NEXT:    mov.b32 %f1, %r2;
+; SM70-NEXT:    ld.param.u16 %r3, [test_maxnum_param_0];
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f2, %r4;
+; SM70-NEXT:    max.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r5, %f3;
+; SM70-NEXT:    bfe.u32 %r6, %r5, 16, 1;
+; SM70-NEXT:    add.s32 %r7, %r6, %r5;
+; SM70-NEXT:    add.s32 %r8, %r7, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r9, %r5, 4194304;
+; SM70-NEXT:    selp.b32 %r10, %r9, %r8, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs1}, %r10; }
+; SM70-NEXT:    st.param.b16 [func_retval0+0], %rs1;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_maxnum(
+; SM80:       {
+; SM80-NEXT:    .reg .b16 %rs<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b16 %rs1, [test_maxnum_param_0];
+; SM80-NEXT:    ld.param.b16 %rs2, [test_maxnum_param_1];
+; SM80-NEXT:    max.bf16 %rs3, %rs1, %rs2;
+; SM80-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_maxnum(
+; SM90:       {
+; SM90-NEXT:    .reg .b16 %rs<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b16 %rs1, [test_maxnum_param_0];
+; SM90-NEXT:    ld.param.b16 %rs2, [test_maxnum_param_1];
+; SM90-NEXT:    max.bf16 %rs3, %rs1, %rs2;
+; SM90-NEXT:    st.param.b16 [func_retval0+0], %rs3;
+; SM90-NEXT:    ret;
+  %r = call bfloat @llvm.maxnum.bf16(bfloat %a, bfloat %b)
+  ret bfloat %r
+}
+
+define <2 x bfloat> @test_maximum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
+; SM70-LABEL: test_maximum_v2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<11>;
+; SM70-NEXT:    .reg .b16 %rs<21>;
+; SM70-NEXT:    .reg .b32 %r<16>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    setp.gt.f32 %p1, %f2, %f1;
+; SM70-NEXT:    selp.b16 %rs7, %rs5, %rs2, %p1;
+; SM70-NEXT:    setp.nan.f32 %p2, %f2, %f1;
+; SM70-NEXT:    selp.b16 %rs8, 0x7FC0, %rs7, %p2;
+; SM70-NEXT:    setp.eq.s16 %p3, %rs5, 0;
+; SM70-NEXT:    selp.b16 %rs9, %rs5, %rs8, %p3;
+; SM70-NEXT:    setp.eq.s16 %p4, %rs2, 0;
+; SM70-NEXT:    selp.b16 %rs10, %rs2, %rs9, %p4;
+; SM70-NEXT:    cvt.u32.u16 %r7, %rs8;
+; SM70-NEXT:    shl.b32 %r8, %r7, 16;
+; SM70-NEXT:    mov.b32 %f3, %r8;
+; SM70-NEXT:    setp.eq.f32 %p5, %f3, 0f00000000;
+; SM70-NEXT:    selp.b16 %rs12, %rs10, %rs8, %p5;
+; SM70-NEXT:    cvt.u32.u16 %r9, %rs1;
+; SM70-NEXT:    shl.b32 %r10, %r9, 16;
+; SM70-NEXT:    mov.b32 %f4, %r10;
+; SM70-NEXT:    cvt.u32.u16 %r11, %rs4;
+; SM70-NEXT:    shl.b32 %r12, %r11, 16;
+; SM70-NEXT:    mov.b32 %f5, %r12;
+; SM70-NEXT:    setp.gt.f32 %p6, %f5, %f4;
+; SM70-NEXT:    selp.b16 %rs15, %rs4, %rs1, %p6;
+; SM70-NEXT:    setp.nan.f32 %p7, %f5, %f4;
+; SM70-NEXT:    selp.b16 %rs16, 0x7FC0, %rs15, %p7;
+; SM70-NEXT:    setp.eq.s16 %p8, %rs4, 0;
+; SM70-NEXT:    selp.b16 %rs17, %rs4, %rs16, %p8;
+; SM70-NEXT:    setp.eq.s16 %p9, %rs1, 0;
+; SM70-NEXT:    selp.b16 %rs18, %rs1, %rs17, %p9;
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs16;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f6, %r14;
+; SM70-NEXT:    setp.eq.f32 %p10, %f6, 0f00000000;
+; SM70-NEXT:    selp.b16 %rs20, %rs18, %rs16, %p10;
+; SM70-NEXT:    mov.b32 %r15, {%rs20, %rs12};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r15;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_maximum_v2(
+; SM80:       {
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_1];
+; SM80-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_0];
+; SM80-NEXT:    max.NaN.bf16x2 %r3, %r2, %r1;
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_maximum_v2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_maximum_v2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_maximum_v2_param_0];
+; SM90-NEXT:    max.NaN.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
+  %r = call <2 x bfloat> @llvm.maximum.bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+  ret <2 x bfloat> %r
+}
+
+define <2 x bfloat> @test_maxnum_v2(<2 x bfloat> %a, <2 x bfloat> %b) {
+; SM70-LABEL: test_maxnum_v2(
+; SM70:       {
+; SM70-NEXT:    .reg .pred %p<3>;
+; SM70-NEXT:    .reg .b16 %rs<13>;
+; SM70-NEXT:    .reg .b32 %r<24>;
+; SM70-NEXT:    .reg .f32 %f<7>;
+; SM70-EMPTY:
+; SM70-NEXT:  // %bb.0:
+; SM70-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_0];
+; SM70-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_1];
+; SM70-NEXT:    mov.b32 {%rs1, %rs2}, %r2;
+; SM70-NEXT:    cvt.u32.u16 %r3, %rs2;
+; SM70-NEXT:    shl.b32 %r4, %r3, 16;
+; SM70-NEXT:    mov.b32 %f1, %r4;
+; SM70-NEXT:    mov.b32 {%rs4, %rs5}, %r1;
+; SM70-NEXT:    cvt.u32.u16 %r5, %rs5;
+; SM70-NEXT:    shl.b32 %r6, %r5, 16;
+; SM70-NEXT:    mov.b32 %f2, %r6;
+; SM70-NEXT:    max.f32 %f3, %f2, %f1;
+; SM70-NEXT:    mov.b32 %r7, %f3;
+; SM70-NEXT:    bfe.u32 %r8, %r7, 16, 1;
+; SM70-NEXT:    add.s32 %r9, %r8, %r7;
+; SM70-NEXT:    add.s32 %r10, %r9, 32767;
+; SM70-NEXT:    setp.nan.f32 %p1, %f3, %f3;
+; SM70-NEXT:    or.b32 %r11, %r7, 4194304;
+; SM70-NEXT:    selp.b32 %r12, %r11, %r10, %p1;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs7}, %r12; }
+; SM70-NEXT:    cvt.u32.u16 %r13, %rs1;
+; SM70-NEXT:    shl.b32 %r14, %r13, 16;
+; SM70-NEXT:    mov.b32 %f4, %r14;
+; SM70-NEXT:    cvt.u32.u16 %r15, %rs4;
+; SM70-NEXT:    shl.b32 %r16, %r15, 16;
+; SM70-NEXT:    mov.b32 %f5, %r16;
+; SM70-NEXT:    max.f32 %f6, %f5, %f4;
+; SM70-NEXT:    mov.b32 %r17, %f6;
+; SM70-NEXT:    bfe.u32 %r18, %r17, 16, 1;
+; SM70-NEXT:    add.s32 %r19, %r18, %r17;
+; SM70-NEXT:    add.s32 %r20, %r19, 32767;
+; SM70-NEXT:    setp.nan.f32 %p2, %f6, %f6;
+; SM70-NEXT:    or.b32 %r21, %r17, 4194304;
+; SM70-NEXT:    selp.b32 %r22, %r21, %r20, %p2;
+; SM70-NEXT:    { .reg .b16 tmp; mov.b32 {tmp, %rs11}, %r22; }
+; SM70-NEXT:    mov.b32 %r23, {%rs11, %rs7};
+; SM70-NEXT:    st.param.b32 [func_retval0+0], %r23;
+; SM70-NEXT:    ret;
+;
+; SM80-LABEL: test_maxnum_v2(
+; SM80:       {
+; SM80-NEXT:    .reg .b32 %r<4>;
+; SM80-EMPTY:
+; SM80-NEXT:  // %bb.0:
+; SM80-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_1];
+; SM80-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_0];
+; SM80-NEXT:    max.bf16x2 %r3, %r2, %r1;
+; SM80-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM80-NEXT:    ret;
+;
+; SM90-LABEL: test_maxnum_v2(
+; SM90:       {
+; SM90-NEXT:    .reg .b32 %r<4>;
+; SM90-EMPTY:
+; SM90-NEXT:  // %bb.0:
+; SM90-NEXT:    ld.param.b32 %r1, [test_maxnum_v2_param_1];
+; SM90-NEXT:    ld.param.b32 %r2, [test_maxnum_v2_param_0];
+; SM90-NEXT:    max.bf16x2 %r3, %r2, %r1;
+; SM90-NEXT:    st.param.b32 [func_retval0+0], %r3;
+; SM90-NEXT:    ret;
+  %r = call <2 x bfloat> @llvm.maxnum.v2bf16(<2 x bfloat> %a, <2 x bfloat> %b)
+  ret <2 x bfloat> %r
+}


        


More information about the llvm-commits mailing list