[llvm] a436e36 - [AArch64][GlobalISel] Legalization for bf16 fptosi.sat/fptoui.sat (#209378)

[via llvm-commits]

Author: David Green
Date: 2026-07-14T06:27:03Z
New Revision: a436e36f5e57e8daabee71fd0cb649c1633e314f

URL: https://github.com/llvm/llvm-project/commit/a436e36f5e57e8daabee71fd0cb649c1633e314f
DIFF: https://github.com/llvm/llvm-project/commit/a436e36f5e57e8daabee71fd0cb649c1633e314f.diff

LOG: [AArch64][GlobalISel] Legalization for bf16 fptosi.sat/fptoui.sat (#209378)

This is the equivalent change to #209206 for fptoi.sat.

Added: 
    

Modified: 
    llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
    llvm/test/CodeGen/AArch64/bf16-instructions.ll
    llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
    llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
    llvm/test/CodeGen/AArch64/round-fptosi-sat-scalar.ll
    llvm/test/CodeGen/AArch64/round-fptoui-sat-scalar.ll

Removed: 
    


################################################################################
diff  --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index df8da27e6769d..387ed79893d3c 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -985,7 +985,12 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       .moreElementsToNextPow2(0)
       .widenScalarToNextPow2(0, /*MinSize=*/32)
       .minScalar(0, s32)
-      .widenScalarOrEltToNextPow2OrMinSize(1, /*MinSize=*/HasFP16 ? 16 : 32)
+      .widenScalarIf(
+          [HasFP16](const LegalityQuery &Query) {
+            return (!HasFP16 && Query.Types[1].getScalarType().isFloat16()) ||
+                   Query.Types[1].getScalarType().isBFloat16();
+          },
+          changeElementTo(1, f32))
       .widenScalarIf(
           [=](const LegalityQuery &Query) {
             unsigned ITySize = Query.Types[0].getScalarSizeInBits();

diff  --git a/llvm/test/CodeGen/AArch64/bf16-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
index eb902d36fbf74..9cd31ca7cc99f 100644
--- a/llvm/test/CodeGen/AArch64/bf16-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-instructions.ll
@@ -4,15 +4,7 @@
 ; RUN: llc < %s -mtriple aarch64 -mattr=-bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
 ; RUN: llc < %s -mtriple aarch64 -mattr=+bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-BF16-GI
 
-; CHECK-CVT-GI:       warning: Instruction selection used fallback path for test_fptosi_sat_i8
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i16
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i32
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i64
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i8
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i16
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i32
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i64
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_uitofp_i32
+; CHECK-CVT-GI:       warning: Instruction selection used fallback path for test_uitofp_i32
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_uitofp_i64
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i32
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i64
@@ -21,15 +13,7 @@
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fpext_double
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fpext_double_fast
 ;
-; CHECK-BF16-GI:       warning: Instruction selection used fallback path for test_fptosi_sat_i8
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i16
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i32
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i64
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i8
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i16
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i32
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i64
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_uitofp_i32
+; CHECK-BF16-GI:       warning: Instruction selection used fallback path for test_uitofp_i32
 ; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_uitofp_i64
 ; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i32
 ; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i64

diff  --git a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
index 78b48901673b6..ea262467e66fe 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v4-instructions.ll
@@ -4,15 +4,7 @@
 ; RUN: llc < %s -mtriple aarch64 -mattr=-bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
 ; RUN: llc < %s -mtriple aarch64 -mattr=+bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-BF16-GI
 
-; CHECK-CVT-GI:       warning: Instruction selection used fallback path for test_fptosi_sat_i8
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i16
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i32
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i64
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i8
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i16
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i32
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i64
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i8
+; CHECK-CVT-GI:       warning: Instruction selection used fallback path for test_sitofp_i8
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i16
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i32
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i64
@@ -21,15 +13,7 @@
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_uitofp_i32
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_uitofp_i64
 ;
-; CHECK-BF16-GI:       warning: Instruction selection used fallback path for test_fptosi_sat_i8
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i16
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i32
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i64
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i8
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i16
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i32
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i64
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i8
+; CHECK-BF16-GI:       warning: Instruction selection used fallback path for test_sitofp_i8
 ; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i16
 ; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i32
 ; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i64
@@ -1361,16 +1345,49 @@ define <4 x i64> @test_fptoui_i64(<4 x bfloat> %a) #0 {
 }
 
 define <4 x i8> @test_fptosi_sat_i8(<4 x bfloat> %a) {
-; CHECK-LABEL: test_fptosi_sat_i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    movi v1.4s, #127
-; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-NEXT:    smin v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    mvni v1.4s, #127
-; CHECK-NEXT:    smax v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    xtn v0.4h, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: test_fptosi_sat_i8:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT:    movi v1.4s, #127
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    smin v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT:    mvni v1.4s, #127
+; CHECK-CVT-SD-NEXT:    smax v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-BF16-SD-LABEL: test_fptosi_sat_i8:
+; CHECK-BF16-SD:       // %bb.0:
+; CHECK-BF16-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT:    movi v1.4s, #127
+; CHECK-BF16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-BF16-SD-NEXT:    smin v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT:    mvni v1.4s, #127
+; CHECK-BF16-SD-NEXT:    smax v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT:    xtn v0.4h, v0.4s
+; CHECK-BF16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fptosi_sat_i8:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT:    movi v1.4s, #127
+; CHECK-CVT-GI-NEXT:    mvni v2.4s, #127
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    smin v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT:    smax v0.4s, v0.4s, v2.4s
+; CHECK-CVT-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-BF16-GI-LABEL: test_fptosi_sat_i8:
+; CHECK-BF16-GI:       // %bb.0:
+; CHECK-BF16-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT:    movi v1.4s, #127
+; CHECK-BF16-GI-NEXT:    mvni v2.4s, #127
+; CHECK-BF16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-BF16-GI-NEXT:    smin v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT:    smax v0.4s, v0.4s, v2.4s
+; CHECK-BF16-GI-NEXT:    xtn v0.4h, v0.4s
+; CHECK-BF16-GI-NEXT:    ret
   %i = call <4 x i8> @llvm.fptosi.sat.v4i8.v4bf16(<4 x bfloat> %a)
   ret <4 x i8> %i
 }
@@ -1397,25 +1414,63 @@ define <4 x i32> @test_fptosi_sat_i32(<4 x bfloat> %a) {
 }
 
 define <4 x i64> @test_fptosi_sat_i64(<4 x bfloat> %a) {
-; CHECK-LABEL: test_fptosi_sat_i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov h1, v0.h[2]
-; CHECK-NEXT:    mov h2, v0.h[1]
-; CHECK-NEXT:    mov h3, v0.h[3]
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcvtzs x8, s0
-; CHECK-NEXT:    fcvtzs x9, s1
-; CHECK-NEXT:    fcvtzs x10, s2
-; CHECK-NEXT:    fcvtzs x11, s3
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    mov v0.d[1], x10
-; CHECK-NEXT:    mov v1.d[1], x11
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: test_fptosi_sat_i64:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT:    mov h1, v0.h[2]
+; CHECK-CVT-SD-NEXT:    mov h2, v0.h[1]
+; CHECK-CVT-SD-NEXT:    mov h3, v0.h[3]
+; CHECK-CVT-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT:    fcvtzs x8, s0
+; CHECK-CVT-SD-NEXT:    fcvtzs x9, s1
+; CHECK-CVT-SD-NEXT:    fcvtzs x10, s2
+; CHECK-CVT-SD-NEXT:    fcvtzs x11, s3
+; CHECK-CVT-SD-NEXT:    fmov d0, x8
+; CHECK-CVT-SD-NEXT:    fmov d1, x9
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], x10
+; CHECK-CVT-SD-NEXT:    mov v1.d[1], x11
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-BF16-SD-LABEL: test_fptosi_sat_i64:
+; CHECK-BF16-SD:       // %bb.0:
+; CHECK-BF16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-BF16-SD-NEXT:    mov h1, v0.h[2]
+; CHECK-BF16-SD-NEXT:    mov h2, v0.h[1]
+; CHECK-BF16-SD-NEXT:    mov h3, v0.h[3]
+; CHECK-BF16-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT:    fcvtzs x8, s0
+; CHECK-BF16-SD-NEXT:    fcvtzs x9, s1
+; CHECK-BF16-SD-NEXT:    fcvtzs x10, s2
+; CHECK-BF16-SD-NEXT:    fcvtzs x11, s3
+; CHECK-BF16-SD-NEXT:    fmov d0, x8
+; CHECK-BF16-SD-NEXT:    fmov d1, x9
+; CHECK-BF16-SD-NEXT:    mov v0.d[1], x10
+; CHECK-BF16-SD-NEXT:    mov v1.d[1], x11
+; CHECK-BF16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fptosi_sat_i64:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v2.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.2d, v1.2d
+; CHECK-CVT-GI-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-BF16-GI-LABEL: test_fptosi_sat_i64:
+; CHECK-BF16-GI:       // %bb.0:
+; CHECK-BF16-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-BF16-GI-NEXT:    fcvtl2 v2.2d, v0.4s
+; CHECK-BF16-GI-NEXT:    fcvtzs v0.2d, v1.2d
+; CHECK-BF16-GI-NEXT:    fcvtzs v1.2d, v2.2d
+; CHECK-BF16-GI-NEXT:    ret
   %i = call <4 x i64> @llvm.fptosi.sat.v4i64.v4bf16(<4 x bfloat> %a)
   ret <4 x i64> %i
 }
@@ -1455,25 +1510,63 @@ define <4 x i32> @test_fptoui_sat_i32(<4 x bfloat> %a) {
 }
 
 define <4 x i64> @test_fptoui_sat_i64(<4 x bfloat> %a) {
-; CHECK-LABEL: test_fptoui_sat_i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-NEXT:    mov h1, v0.h[2]
-; CHECK-NEXT:    mov h2, v0.h[1]
-; CHECK-NEXT:    mov h3, v0.h[3]
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    fcvtzu x8, s0
-; CHECK-NEXT:    fcvtzu x9, s1
-; CHECK-NEXT:    fcvtzu x10, s2
-; CHECK-NEXT:    fcvtzu x11, s3
-; CHECK-NEXT:    fmov d0, x8
-; CHECK-NEXT:    fmov d1, x9
-; CHECK-NEXT:    mov v0.d[1], x10
-; CHECK-NEXT:    mov v1.d[1], x11
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: test_fptoui_sat_i64:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-CVT-SD-NEXT:    mov h1, v0.h[2]
+; CHECK-CVT-SD-NEXT:    mov h2, v0.h[1]
+; CHECK-CVT-SD-NEXT:    mov h3, v0.h[3]
+; CHECK-CVT-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT:    fcvtzu x8, s0
+; CHECK-CVT-SD-NEXT:    fcvtzu x9, s1
+; CHECK-CVT-SD-NEXT:    fcvtzu x10, s2
+; CHECK-CVT-SD-NEXT:    fcvtzu x11, s3
+; CHECK-CVT-SD-NEXT:    fmov d0, x8
+; CHECK-CVT-SD-NEXT:    fmov d1, x9
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], x10
+; CHECK-CVT-SD-NEXT:    mov v1.d[1], x11
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-BF16-SD-LABEL: test_fptoui_sat_i64:
+; CHECK-BF16-SD:       // %bb.0:
+; CHECK-BF16-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-BF16-SD-NEXT:    mov h1, v0.h[2]
+; CHECK-BF16-SD-NEXT:    mov h2, v0.h[1]
+; CHECK-BF16-SD-NEXT:    mov h3, v0.h[3]
+; CHECK-BF16-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT:    fcvtzu x8, s0
+; CHECK-BF16-SD-NEXT:    fcvtzu x9, s1
+; CHECK-BF16-SD-NEXT:    fcvtzu x10, s2
+; CHECK-BF16-SD-NEXT:    fcvtzu x11, s3
+; CHECK-BF16-SD-NEXT:    fmov d0, x8
+; CHECK-BF16-SD-NEXT:    fmov d1, x9
+; CHECK-BF16-SD-NEXT:    mov v0.d[1], x10
+; CHECK-BF16-SD-NEXT:    mov v1.d[1], x11
+; CHECK-BF16-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fptoui_sat_i64:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v2.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    fcvtzu v0.2d, v1.2d
+; CHECK-CVT-GI-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-BF16-GI-LABEL: test_fptoui_sat_i64:
+; CHECK-BF16-GI:       // %bb.0:
+; CHECK-BF16-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT:    fcvtl v1.2d, v0.2s
+; CHECK-BF16-GI-NEXT:    fcvtl2 v2.2d, v0.4s
+; CHECK-BF16-GI-NEXT:    fcvtzu v0.2d, v1.2d
+; CHECK-BF16-GI-NEXT:    fcvtzu v1.2d, v2.2d
+; CHECK-BF16-GI-NEXT:    ret
   %i = call <4 x i64> @llvm.fptoui.sat.v4i64.v4bf16(<4 x bfloat> %a)
   ret <4 x i64> %i
 }

diff  --git a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
index 382576f94dfdd..85faf1facc264 100644
--- a/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
+++ b/llvm/test/CodeGen/AArch64/bf16-v8-instructions.ll
@@ -5,15 +5,7 @@
 ; RUN: llc < %s -mtriple aarch64 -mattr=-bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT,CHECK-CVT-GI
 ; RUN: llc < %s -mtriple aarch64 -mattr=+bf16 -global-isel -global-isel-abort=2 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-BF16,CHECK-BF16-GI
 
-; CHECK-CVT-GI:       warning: Instruction selection used fallback path for test_fptosi_sat_i8
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i16
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i32
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i64
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i8
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i16
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i32
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i64
-; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i8
+; CHECK-CVT-GI:       warning: Instruction selection used fallback path for test_sitofp_i8
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_v16i8
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i16
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i32
@@ -24,15 +16,7 @@
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_uitofp_i32
 ; CHECK-CVT-GI-NEXT:  warning: Instruction selection used fallback path for test_uitofp_i64
 ;
-; CHECK-BF16-GI:       warning: Instruction selection used fallback path for test_fptosi_sat_i8
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i16
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i32
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptosi_sat_i64
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i8
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i16
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i32
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_fptoui_sat_i64
-; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i8
+; CHECK-BF16-GI:       warning: Instruction selection used fallback path for test_sitofp_i8
 ; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_v16i8
 ; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i16
 ; CHECK-BF16-GI-NEXT:  warning: Instruction selection used fallback path for test_sitofp_i32
@@ -3056,168 +3040,624 @@ define <8 x i64> @test_fptoui_i64(<8 x bfloat> %a) #0 {
 }
 
 define <8 x i8> @test_fptosi_sat_i8(<8 x bfloat> %a) {
-; CHECK-LABEL: test_fptosi_sat_i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    shll2 v2.4s, v0.8h, #16
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    movi v1.4s, #127
-; CHECK-NEXT:    mvni v3.4s, #127
-; CHECK-NEXT:    fcvtzs v2.4s, v2.4s
-; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-NEXT:    smin v2.4s, v2.4s, v1.4s
-; CHECK-NEXT:    smin v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    smax v1.4s, v2.4s, v3.4s
-; CHECK-NEXT:    smax v0.4s, v0.4s, v3.4s
-; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
-; CHECK-NEXT:    xtn v0.8b, v0.8h
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: test_fptosi_sat_i8:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT:    movi v1.4s, #127
+; CHECK-CVT-SD-NEXT:    mvni v3.4s, #127
+; CHECK-CVT-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    smin v2.4s, v2.4s, v1.4s
+; CHECK-CVT-SD-NEXT:    smin v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT:    smax v1.4s, v2.4s, v3.4s
+; CHECK-CVT-SD-NEXT:    smax v0.4s, v0.4s, v3.4s
+; CHECK-CVT-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-CVT-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-BF16-SD-LABEL: test_fptosi_sat_i8:
+; CHECK-BF16-SD:       // %bb.0:
+; CHECK-BF16-SD-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT:    movi v1.4s, #127
+; CHECK-BF16-SD-NEXT:    mvni v3.4s, #127
+; CHECK-BF16-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-BF16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-BF16-SD-NEXT:    smin v2.4s, v2.4s, v1.4s
+; CHECK-BF16-SD-NEXT:    smin v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT:    smax v1.4s, v2.4s, v3.4s
+; CHECK-BF16-SD-NEXT:    smax v0.4s, v0.4s, v3.4s
+; CHECK-BF16-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BF16-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-BF16-SD-NEXT:    ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_fptosi_sat_i8:
+; CHECK-BF16SVE-SD:       // %bb.0:
+; CHECK-BF16SVE-SD-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    movi v1.4s, #127
+; CHECK-BF16SVE-SD-NEXT:    mvni v3.4s, #127
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-BF16SVE-SD-NEXT:    smin v2.4s, v2.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT:    smin v0.4s, v0.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT:    smax v1.4s, v2.4s, v3.4s
+; CHECK-BF16SVE-SD-NEXT:    smax v0.4s, v0.4s, v3.4s
+; CHECK-BF16SVE-SD-NEXT:    uzp1 v0.8h, v0.8h, v1.8h
+; CHECK-BF16SVE-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-BF16SVE-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fptosi_sat_i8:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    shll v2.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT:    movi v1.4s, #127
+; CHECK-CVT-GI-NEXT:    mvni v3.4s, #127
+; CHECK-CVT-GI-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    smin v2.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT:    smin v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT:    smax v1.4s, v2.4s, v3.4s
+; CHECK-CVT-GI-NEXT:    smax v0.4s, v0.4s, v3.4s
+; CHECK-CVT-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-CVT-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-BF16-GI-LABEL: test_fptosi_sat_i8:
+; CHECK-BF16-GI:       // %bb.0:
+; CHECK-BF16-GI-NEXT:    shll v2.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT:    movi v1.4s, #127
+; CHECK-BF16-GI-NEXT:    mvni v3.4s, #127
+; CHECK-BF16-GI-NEXT:    fcvtzs v2.4s, v2.4s
+; CHECK-BF16-GI-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-BF16-GI-NEXT:    smin v2.4s, v2.4s, v1.4s
+; CHECK-BF16-GI-NEXT:    smin v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT:    smax v1.4s, v2.4s, v3.4s
+; CHECK-BF16-GI-NEXT:    smax v0.4s, v0.4s, v3.4s
+; CHECK-BF16-GI-NEXT:    uzp1 v0.8h, v1.8h, v0.8h
+; CHECK-BF16-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-BF16-GI-NEXT:    ret
   %i = call <8 x i8> @llvm.fptosi.sat.v8i8.v8bf16(<8 x bfloat> %a)
   ret <8 x i8> %i
 }
 
 define <8 x i16> @test_fptosi_sat_i16(<8 x bfloat> %a) {
-; CHECK-LABEL: test_fptosi_sat_i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    shll v1.4s, v0.4h, #16
-; CHECK-NEXT:    shll2 v2.4s, v0.8h, #16
-; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-NEXT:    sqxtn v0.4h, v1.4s
-; CHECK-NEXT:    fcvtzs v1.4s, v2.4s
-; CHECK-NEXT:    sqxtn2 v0.8h, v1.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: test_fptosi_sat_i16:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-CVT-SD-NEXT:    sqxtn v0.4h, v1.4s
+; CHECK-CVT-SD-NEXT:    fcvtzs v1.4s, v2.4s
+; CHECK-CVT-SD-NEXT:    sqxtn2 v0.8h, v1.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-BF16-SD-LABEL: test_fptosi_sat_i16:
+; CHECK-BF16-SD:       // %bb.0:
+; CHECK-BF16-SD-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT:    sqxtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT:    fcvtzs v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT:    sqxtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT:    ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_fptosi_sat_i16:
+; CHECK-BF16SVE-SD:       // %bb.0:
+; CHECK-BF16SVE-SD-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT:    sqxtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT:    sqxtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fptosi_sat_i16:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT:    fcvtzs v2.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    sqxtn v0.4h, v1.4s
+; CHECK-CVT-GI-NEXT:    sqxtn2 v0.8h, v2.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-BF16-GI-LABEL: test_fptosi_sat_i16:
+; CHECK-BF16-GI:       // %bb.0:
+; CHECK-BF16-GI-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT:    fcvtzs v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT:    sqxtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT:    sqxtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT:    ret
   %i = call <8 x i16> @llvm.fptosi.sat.v8i16.v8bf16(<8 x bfloat> %a)
   ret <8 x i16> %i
 }
 
 define <8 x i32> @test_fptosi_sat_i32(<8 x bfloat> %a) {
-; CHECK-LABEL: test_fptosi_sat_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    shll2 v1.4s, v0.8h, #16
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzs v1.4s, v1.4s
-; CHECK-NEXT:    fcvtzs v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: test_fptosi_sat_i32:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    shll2 v1.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-CVT-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-BF16-SD-LABEL: test_fptosi_sat_i32:
+; CHECK-BF16-SD:       // %bb.0:
+; CHECK-BF16-SD-NEXT:    shll2 v1.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-BF16-SD-NEXT:    ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_fptosi_sat_i32:
+; CHECK-BF16SVE-SD:       // %bb.0:
+; CHECK-BF16SVE-SD-NEXT:    shll2 v1.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs v0.4s, v0.4s
+; CHECK-BF16SVE-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fptosi_sat_i32:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT:    fcvtzs v1.4s, v2.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-BF16-GI-LABEL: test_fptosi_sat_i32:
+; CHECK-BF16-GI:       // %bb.0:
+; CHECK-BF16-GI-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT:    fcvtzs v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT:    fcvtzs v1.4s, v2.4s
+; CHECK-BF16-GI-NEXT:    ret
   %i = call <8 x i32> @llvm.fptosi.sat.v8i32.v8bf16(<8 x bfloat> %a)
   ret <8 x i32> %i
 }
 
 define <8 x i64> @test_fptosi_sat_i64(<8 x bfloat> %a) {
-; CHECK-LABEL: test_fptosi_sat_i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d1, v0.d[1]
-; CHECK-NEXT:    mov h4, v0.h[2]
-; CHECK-NEXT:    mov h3, v0.h[1]
-; CHECK-NEXT:    mov h7, v0.h[3]
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov h2, v1.h[2]
-; CHECK-NEXT:    mov h5, v1.h[1]
-; CHECK-NEXT:    mov h6, v1.h[3]
-; CHECK-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-NEXT:    shll v4.4s, v4.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    shll v7.4s, v7.4h, #16
-; CHECK-NEXT:    fcvtzs x9, s0
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v5.4s, v5.4h, #16
-; CHECK-NEXT:    shll v6.4s, v6.4h, #16
-; CHECK-NEXT:    fcvtzs x8, s1
-; CHECK-NEXT:    fcvtzs x12, s4
-; CHECK-NEXT:    fcvtzs x11, s3
-; CHECK-NEXT:    fcvtzs x15, s7
-; CHECK-NEXT:    fmov d0, x9
-; CHECK-NEXT:    fcvtzs x10, s2
-; CHECK-NEXT:    fcvtzs x13, s5
-; CHECK-NEXT:    fcvtzs x14, s6
-; CHECK-NEXT:    fmov d2, x8
-; CHECK-NEXT:    fmov d1, x12
-; CHECK-NEXT:    mov v0.d[1], x11
-; CHECK-NEXT:    fmov d3, x10
-; CHECK-NEXT:    mov v2.d[1], x13
-; CHECK-NEXT:    mov v1.d[1], x15
-; CHECK-NEXT:    mov v3.d[1], x14
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: test_fptosi_sat_i64:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    mov d1, v0.d[1]
+; CHECK-CVT-SD-NEXT:    mov h4, v0.h[2]
+; CHECK-CVT-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-CVT-SD-NEXT:    mov h7, v0.h[3]
+; CHECK-CVT-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT:    mov h2, v1.h[2]
+; CHECK-CVT-SD-NEXT:    mov h5, v1.h[1]
+; CHECK-CVT-SD-NEXT:    mov h6, v1.h[3]
+; CHECK-CVT-SD-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v4.4s, v4.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v7.4s, v7.4h, #16
+; CHECK-CVT-SD-NEXT:    fcvtzs x9, s0
+; CHECK-CVT-SD-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v5.4s, v5.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v6.4s, v6.4h, #16
+; CHECK-CVT-SD-NEXT:    fcvtzs x8, s1
+; CHECK-CVT-SD-NEXT:    fcvtzs x12, s4
+; CHECK-CVT-SD-NEXT:    fcvtzs x11, s3
+; CHECK-CVT-SD-NEXT:    fcvtzs x15, s7
+; CHECK-CVT-SD-NEXT:    fmov d0, x9
+; CHECK-CVT-SD-NEXT:    fcvtzs x10, s2
+; CHECK-CVT-SD-NEXT:    fcvtzs x13, s5
+; CHECK-CVT-SD-NEXT:    fcvtzs x14, s6
+; CHECK-CVT-SD-NEXT:    fmov d2, x8
+; CHECK-CVT-SD-NEXT:    fmov d1, x12
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], x11
+; CHECK-CVT-SD-NEXT:    fmov d3, x10
+; CHECK-CVT-SD-NEXT:    mov v2.d[1], x13
+; CHECK-CVT-SD-NEXT:    mov v1.d[1], x15
+; CHECK-CVT-SD-NEXT:    mov v3.d[1], x14
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-BF16-SD-LABEL: test_fptosi_sat_i64:
+; CHECK-BF16-SD:       // %bb.0:
+; CHECK-BF16-SD-NEXT:    mov d1, v0.d[1]
+; CHECK-BF16-SD-NEXT:    mov h4, v0.h[2]
+; CHECK-BF16-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-BF16-SD-NEXT:    mov h7, v0.h[3]
+; CHECK-BF16-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT:    mov h2, v1.h[2]
+; CHECK-BF16-SD-NEXT:    mov h5, v1.h[1]
+; CHECK-BF16-SD-NEXT:    mov h6, v1.h[3]
+; CHECK-BF16-SD-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v4.4s, v4.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v7.4s, v7.4h, #16
+; CHECK-BF16-SD-NEXT:    fcvtzs x9, s0
+; CHECK-BF16-SD-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v5.4s, v5.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v6.4s, v6.4h, #16
+; CHECK-BF16-SD-NEXT:    fcvtzs x8, s1
+; CHECK-BF16-SD-NEXT:    fcvtzs x12, s4
+; CHECK-BF16-SD-NEXT:    fcvtzs x11, s3
+; CHECK-BF16-SD-NEXT:    fcvtzs x15, s7
+; CHECK-BF16-SD-NEXT:    fmov d0, x9
+; CHECK-BF16-SD-NEXT:    fcvtzs x10, s2
+; CHECK-BF16-SD-NEXT:    fcvtzs x13, s5
+; CHECK-BF16-SD-NEXT:    fcvtzs x14, s6
+; CHECK-BF16-SD-NEXT:    fmov d2, x8
+; CHECK-BF16-SD-NEXT:    fmov d1, x12
+; CHECK-BF16-SD-NEXT:    mov v0.d[1], x11
+; CHECK-BF16-SD-NEXT:    fmov d3, x10
+; CHECK-BF16-SD-NEXT:    mov v2.d[1], x13
+; CHECK-BF16-SD-NEXT:    mov v1.d[1], x15
+; CHECK-BF16-SD-NEXT:    mov v3.d[1], x14
+; CHECK-BF16-SD-NEXT:    ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_fptosi_sat_i64:
+; CHECK-BF16SVE-SD:       // %bb.0:
+; CHECK-BF16SVE-SD-NEXT:    mov d1, v0.d[1]
+; CHECK-BF16SVE-SD-NEXT:    mov h4, v0.h[2]
+; CHECK-BF16SVE-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-BF16SVE-SD-NEXT:    mov h7, v0.h[3]
+; CHECK-BF16SVE-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    mov h2, v1.h[2]
+; CHECK-BF16SVE-SD-NEXT:    mov h5, v1.h[1]
+; CHECK-BF16SVE-SD-NEXT:    mov h6, v1.h[3]
+; CHECK-BF16SVE-SD-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v4.4s, v4.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v3.4s, v3.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v7.4s, v7.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs x9, s0
+; CHECK-BF16SVE-SD-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v5.4s, v5.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v6.4s, v6.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs x8, s1
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs x12, s4
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs x11, s3
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs x15, s7
+; CHECK-BF16SVE-SD-NEXT:    fmov d0, x9
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs x10, s2
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs x13, s5
+; CHECK-BF16SVE-SD-NEXT:    fcvtzs x14, s6
+; CHECK-BF16SVE-SD-NEXT:    fmov d2, x8
+; CHECK-BF16SVE-SD-NEXT:    fmov d1, x12
+; CHECK-BF16SVE-SD-NEXT:    mov v0.d[1], x11
+; CHECK-BF16SVE-SD-NEXT:    fmov d3, x10
+; CHECK-BF16SVE-SD-NEXT:    mov v2.d[1], x13
+; CHECK-BF16SVE-SD-NEXT:    mov v1.d[1], x15
+; CHECK-BF16SVE-SD-NEXT:    mov v3.d[1], x14
+; CHECK-BF16SVE-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fptosi_sat_i64:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT:    fcvtl v2.2d, v1.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v1.2d, v1.4s
+; CHECK-CVT-GI-NEXT:    fcvtl v3.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v4.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    fcvtzs v0.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-CVT-GI-NEXT:    fcvtzs v2.2d, v3.2d
+; CHECK-CVT-GI-NEXT:    fcvtzs v3.2d, v4.2d
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-BF16-GI-LABEL: test_fptosi_sat_i64:
+; CHECK-BF16-GI:       // %bb.0:
+; CHECK-BF16-GI-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT:    fcvtl v2.2d, v1.2s
+; CHECK-BF16-GI-NEXT:    fcvtl2 v1.2d, v1.4s
+; CHECK-BF16-GI-NEXT:    fcvtl v3.2d, v0.2s
+; CHECK-BF16-GI-NEXT:    fcvtl2 v4.2d, v0.4s
+; CHECK-BF16-GI-NEXT:    fcvtzs v0.2d, v2.2d
+; CHECK-BF16-GI-NEXT:    fcvtzs v1.2d, v1.2d
+; CHECK-BF16-GI-NEXT:    fcvtzs v2.2d, v3.2d
+; CHECK-BF16-GI-NEXT:    fcvtzs v3.2d, v4.2d
+; CHECK-BF16-GI-NEXT:    ret
   %i = call <8 x i64> @llvm.fptosi.sat.v8i64.v8bf16(<8 x bfloat> %a)
   ret <8 x i64> %i
 }
 
 define <8 x i8> @test_fptoui_sat_i8(<8 x bfloat> %a) {
-; CHECK-LABEL: test_fptoui_sat_i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    shll2 v2.4s, v0.8h, #16
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    movi v1.2d, #0x0000ff000000ff
-; CHECK-NEXT:    fcvtzu v2.4s, v2.4s
-; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    umin v2.4s, v2.4s, v1.4s
-; CHECK-NEXT:    umin v0.4s, v0.4s, v1.4s
-; CHECK-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
-; CHECK-NEXT:    xtn v0.8b, v0.8h
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: test_fptoui_sat_i8:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT:    movi v1.2d, #0x0000ff000000ff
+; CHECK-CVT-SD-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-CVT-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    umin v2.4s, v2.4s, v1.4s
+; CHECK-CVT-SD-NEXT:    umin v0.4s, v0.4s, v1.4s
+; CHECK-CVT-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-CVT-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-BF16-SD-LABEL: test_fptoui_sat_i8:
+; CHECK-BF16-SD:       // %bb.0:
+; CHECK-BF16-SD-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT:    movi v1.2d, #0x0000ff000000ff
+; CHECK-BF16-SD-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-BF16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-BF16-SD-NEXT:    umin v2.4s, v2.4s, v1.4s
+; CHECK-BF16-SD-NEXT:    umin v0.4s, v0.4s, v1.4s
+; CHECK-BF16-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-BF16-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-BF16-SD-NEXT:    ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_fptoui_sat_i8:
+; CHECK-BF16SVE-SD:       // %bb.0:
+; CHECK-BF16SVE-SD-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    movi v1.2d, #0x0000ff000000ff
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-BF16SVE-SD-NEXT:    umin v2.4s, v2.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT:    umin v0.4s, v0.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT:    uzp1 v0.8h, v0.8h, v2.8h
+; CHECK-BF16SVE-SD-NEXT:    xtn v0.8b, v0.8h
+; CHECK-BF16SVE-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fptoui_sat_i8:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    shll v2.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT:    movi v1.2d, #0x0000ff000000ff
+; CHECK-CVT-GI-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-CVT-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    umin v2.4s, v2.4s, v1.4s
+; CHECK-CVT-GI-NEXT:    umin v0.4s, v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
+; CHECK-CVT-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-BF16-GI-LABEL: test_fptoui_sat_i8:
+; CHECK-BF16-GI:       // %bb.0:
+; CHECK-BF16-GI-NEXT:    shll v2.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT:    movi v1.2d, #0x0000ff000000ff
+; CHECK-BF16-GI-NEXT:    fcvtzu v2.4s, v2.4s
+; CHECK-BF16-GI-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-BF16-GI-NEXT:    umin v2.4s, v2.4s, v1.4s
+; CHECK-BF16-GI-NEXT:    umin v0.4s, v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT:    uzp1 v0.8h, v2.8h, v0.8h
+; CHECK-BF16-GI-NEXT:    xtn v0.8b, v0.8h
+; CHECK-BF16-GI-NEXT:    ret
   %i = call <8 x i8> @llvm.fptoui.sat.v8i8.v8bf16(<8 x bfloat> %a)
   ret <8 x i8> %i
 }
 
 define <8 x i16> @test_fptoui_sat_i16(<8 x bfloat> %a) {
-; CHECK-LABEL: test_fptoui_sat_i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    shll v1.4s, v0.4h, #16
-; CHECK-NEXT:    shll2 v2.4s, v0.8h, #16
-; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-NEXT:    uqxtn v0.4h, v1.4s
-; CHECK-NEXT:    fcvtzu v1.4s, v2.4s
-; CHECK-NEXT:    uqxtn2 v0.8h, v1.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: test_fptoui_sat_i16:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-CVT-SD-NEXT:    uqxtn v0.4h, v1.4s
+; CHECK-CVT-SD-NEXT:    fcvtzu v1.4s, v2.4s
+; CHECK-CVT-SD-NEXT:    uqxtn2 v0.8h, v1.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-BF16-SD-LABEL: test_fptoui_sat_i16:
+; CHECK-BF16-SD:       // %bb.0:
+; CHECK-BF16-SD-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT:    uqxtn v0.4h, v1.4s
+; CHECK-BF16-SD-NEXT:    fcvtzu v1.4s, v2.4s
+; CHECK-BF16-SD-NEXT:    uqxtn2 v0.8h, v1.4s
+; CHECK-BF16-SD-NEXT:    ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_fptoui_sat_i16:
+; CHECK-BF16SVE-SD:       // %bb.0:
+; CHECK-BF16SVE-SD-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT:    uqxtn v0.4h, v1.4s
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu v1.4s, v2.4s
+; CHECK-BF16SVE-SD-NEXT:    uqxtn2 v0.8h, v1.4s
+; CHECK-BF16SVE-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fptoui_sat_i16:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-CVT-GI-NEXT:    fcvtzu v2.4s, v0.4s
+; CHECK-CVT-GI-NEXT:    uqxtn v0.4h, v1.4s
+; CHECK-CVT-GI-NEXT:    uqxtn2 v0.8h, v2.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-BF16-GI-LABEL: test_fptoui_sat_i16:
+; CHECK-BF16-GI:       // %bb.0:
+; CHECK-BF16-GI-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-BF16-GI-NEXT:    fcvtzu v2.4s, v0.4s
+; CHECK-BF16-GI-NEXT:    uqxtn v0.4h, v1.4s
+; CHECK-BF16-GI-NEXT:    uqxtn2 v0.8h, v2.4s
+; CHECK-BF16-GI-NEXT:    ret
   %i = call <8 x i16> @llvm.fptoui.sat.v8i16.v8bf16(<8 x bfloat> %a)
   ret <8 x i16> %i
 }
 
 define <8 x i32> @test_fptoui_sat_i32(<8 x bfloat> %a) {
-; CHECK-LABEL: test_fptoui_sat_i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    shll2 v1.4s, v0.8h, #16
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzu v1.4s, v1.4s
-; CHECK-NEXT:    fcvtzu v0.4s, v0.4s
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: test_fptoui_sat_i32:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    shll2 v1.4s, v0.8h, #16
+; CHECK-CVT-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-CVT-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-BF16-SD-LABEL: test_fptoui_sat_i32:
+; CHECK-BF16-SD:       // %bb.0:
+; CHECK-BF16-SD-NEXT:    shll2 v1.4s, v0.8h, #16
+; CHECK-BF16-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-BF16-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-BF16-SD-NEXT:    ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_fptoui_sat_i32:
+; CHECK-BF16SVE-SD:       // %bb.0:
+; CHECK-BF16SVE-SD-NEXT:    shll2 v1.4s, v0.8h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu v1.4s, v1.4s
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu v0.4s, v0.4s
+; CHECK-BF16SVE-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fptoui_sat_i32:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT:    fcvtzu v0.4s, v1.4s
+; CHECK-CVT-GI-NEXT:    fcvtzu v1.4s, v2.4s
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-BF16-GI-LABEL: test_fptoui_sat_i32:
+; CHECK-BF16-GI:       // %bb.0:
+; CHECK-BF16-GI-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT:    shll2 v2.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT:    fcvtzu v0.4s, v1.4s
+; CHECK-BF16-GI-NEXT:    fcvtzu v1.4s, v2.4s
+; CHECK-BF16-GI-NEXT:    ret
   %i = call <8 x i32> @llvm.fptoui.sat.v8i32.v8bf16(<8 x bfloat> %a)
   ret <8 x i32> %i
 }
 
 define <8 x i64> @test_fptoui_sat_i64(<8 x bfloat> %a) {
-; CHECK-LABEL: test_fptoui_sat_i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov d1, v0.d[1]
-; CHECK-NEXT:    mov h4, v0.h[2]
-; CHECK-NEXT:    mov h3, v0.h[1]
-; CHECK-NEXT:    mov h7, v0.h[3]
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    mov h2, v1.h[2]
-; CHECK-NEXT:    mov h5, v1.h[1]
-; CHECK-NEXT:    mov h6, v1.h[3]
-; CHECK-NEXT:    shll v1.4s, v1.4h, #16
-; CHECK-NEXT:    shll v4.4s, v4.4h, #16
-; CHECK-NEXT:    shll v3.4s, v3.4h, #16
-; CHECK-NEXT:    shll v7.4s, v7.4h, #16
-; CHECK-NEXT:    fcvtzu x9, s0
-; CHECK-NEXT:    shll v2.4s, v2.4h, #16
-; CHECK-NEXT:    shll v5.4s, v5.4h, #16
-; CHECK-NEXT:    shll v6.4s, v6.4h, #16
-; CHECK-NEXT:    fcvtzu x8, s1
-; CHECK-NEXT:    fcvtzu x12, s4
-; CHECK-NEXT:    fcvtzu x11, s3
-; CHECK-NEXT:    fcvtzu x15, s7
-; CHECK-NEXT:    fmov d0, x9
-; CHECK-NEXT:    fcvtzu x10, s2
-; CHECK-NEXT:    fcvtzu x13, s5
-; CHECK-NEXT:    fcvtzu x14, s6
-; CHECK-NEXT:    fmov d2, x8
-; CHECK-NEXT:    fmov d1, x12
-; CHECK-NEXT:    mov v0.d[1], x11
-; CHECK-NEXT:    fmov d3, x10
-; CHECK-NEXT:    mov v2.d[1], x13
-; CHECK-NEXT:    mov v1.d[1], x15
-; CHECK-NEXT:    mov v3.d[1], x14
-; CHECK-NEXT:    ret
+; CHECK-CVT-SD-LABEL: test_fptoui_sat_i64:
+; CHECK-CVT-SD:       // %bb.0:
+; CHECK-CVT-SD-NEXT:    mov d1, v0.d[1]
+; CHECK-CVT-SD-NEXT:    mov h4, v0.h[2]
+; CHECK-CVT-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-CVT-SD-NEXT:    mov h7, v0.h[3]
+; CHECK-CVT-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-SD-NEXT:    mov h2, v1.h[2]
+; CHECK-CVT-SD-NEXT:    mov h5, v1.h[1]
+; CHECK-CVT-SD-NEXT:    mov h6, v1.h[3]
+; CHECK-CVT-SD-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v4.4s, v4.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v3.4s, v3.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v7.4s, v7.4h, #16
+; CHECK-CVT-SD-NEXT:    fcvtzu x9, s0
+; CHECK-CVT-SD-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v5.4s, v5.4h, #16
+; CHECK-CVT-SD-NEXT:    shll v6.4s, v6.4h, #16
+; CHECK-CVT-SD-NEXT:    fcvtzu x8, s1
+; CHECK-CVT-SD-NEXT:    fcvtzu x12, s4
+; CHECK-CVT-SD-NEXT:    fcvtzu x11, s3
+; CHECK-CVT-SD-NEXT:    fcvtzu x15, s7
+; CHECK-CVT-SD-NEXT:    fmov d0, x9
+; CHECK-CVT-SD-NEXT:    fcvtzu x10, s2
+; CHECK-CVT-SD-NEXT:    fcvtzu x13, s5
+; CHECK-CVT-SD-NEXT:    fcvtzu x14, s6
+; CHECK-CVT-SD-NEXT:    fmov d2, x8
+; CHECK-CVT-SD-NEXT:    fmov d1, x12
+; CHECK-CVT-SD-NEXT:    mov v0.d[1], x11
+; CHECK-CVT-SD-NEXT:    fmov d3, x10
+; CHECK-CVT-SD-NEXT:    mov v2.d[1], x13
+; CHECK-CVT-SD-NEXT:    mov v1.d[1], x15
+; CHECK-CVT-SD-NEXT:    mov v3.d[1], x14
+; CHECK-CVT-SD-NEXT:    ret
+;
+; CHECK-BF16-SD-LABEL: test_fptoui_sat_i64:
+; CHECK-BF16-SD:       // %bb.0:
+; CHECK-BF16-SD-NEXT:    mov d1, v0.d[1]
+; CHECK-BF16-SD-NEXT:    mov h4, v0.h[2]
+; CHECK-BF16-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-BF16-SD-NEXT:    mov h7, v0.h[3]
+; CHECK-BF16-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16-SD-NEXT:    mov h2, v1.h[2]
+; CHECK-BF16-SD-NEXT:    mov h5, v1.h[1]
+; CHECK-BF16-SD-NEXT:    mov h6, v1.h[3]
+; CHECK-BF16-SD-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v4.4s, v4.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v3.4s, v3.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v7.4s, v7.4h, #16
+; CHECK-BF16-SD-NEXT:    fcvtzu x9, s0
+; CHECK-BF16-SD-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v5.4s, v5.4h, #16
+; CHECK-BF16-SD-NEXT:    shll v6.4s, v6.4h, #16
+; CHECK-BF16-SD-NEXT:    fcvtzu x8, s1
+; CHECK-BF16-SD-NEXT:    fcvtzu x12, s4
+; CHECK-BF16-SD-NEXT:    fcvtzu x11, s3
+; CHECK-BF16-SD-NEXT:    fcvtzu x15, s7
+; CHECK-BF16-SD-NEXT:    fmov d0, x9
+; CHECK-BF16-SD-NEXT:    fcvtzu x10, s2
+; CHECK-BF16-SD-NEXT:    fcvtzu x13, s5
+; CHECK-BF16-SD-NEXT:    fcvtzu x14, s6
+; CHECK-BF16-SD-NEXT:    fmov d2, x8
+; CHECK-BF16-SD-NEXT:    fmov d1, x12
+; CHECK-BF16-SD-NEXT:    mov v0.d[1], x11
+; CHECK-BF16-SD-NEXT:    fmov d3, x10
+; CHECK-BF16-SD-NEXT:    mov v2.d[1], x13
+; CHECK-BF16-SD-NEXT:    mov v1.d[1], x15
+; CHECK-BF16-SD-NEXT:    mov v3.d[1], x14
+; CHECK-BF16-SD-NEXT:    ret
+;
+; CHECK-BF16SVE-SD-LABEL: test_fptoui_sat_i64:
+; CHECK-BF16SVE-SD:       // %bb.0:
+; CHECK-BF16SVE-SD-NEXT:    mov d1, v0.d[1]
+; CHECK-BF16SVE-SD-NEXT:    mov h4, v0.h[2]
+; CHECK-BF16SVE-SD-NEXT:    mov h3, v0.h[1]
+; CHECK-BF16SVE-SD-NEXT:    mov h7, v0.h[3]
+; CHECK-BF16SVE-SD-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    mov h2, v1.h[2]
+; CHECK-BF16SVE-SD-NEXT:    mov h5, v1.h[1]
+; CHECK-BF16SVE-SD-NEXT:    mov h6, v1.h[3]
+; CHECK-BF16SVE-SD-NEXT:    shll v1.4s, v1.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v4.4s, v4.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v3.4s, v3.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v7.4s, v7.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu x9, s0
+; CHECK-BF16SVE-SD-NEXT:    shll v2.4s, v2.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v5.4s, v5.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    shll v6.4s, v6.4h, #16
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu x8, s1
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu x12, s4
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu x11, s3
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu x15, s7
+; CHECK-BF16SVE-SD-NEXT:    fmov d0, x9
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu x10, s2
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu x13, s5
+; CHECK-BF16SVE-SD-NEXT:    fcvtzu x14, s6
+; CHECK-BF16SVE-SD-NEXT:    fmov d2, x8
+; CHECK-BF16SVE-SD-NEXT:    fmov d1, x12
+; CHECK-BF16SVE-SD-NEXT:    mov v0.d[1], x11
+; CHECK-BF16SVE-SD-NEXT:    fmov d3, x10
+; CHECK-BF16SVE-SD-NEXT:    mov v2.d[1], x13
+; CHECK-BF16SVE-SD-NEXT:    mov v1.d[1], x15
+; CHECK-BF16SVE-SD-NEXT:    mov v3.d[1], x14
+; CHECK-BF16SVE-SD-NEXT:    ret
+;
+; CHECK-CVT-GI-LABEL: test_fptoui_sat_i64:
+; CHECK-CVT-GI:       // %bb.0:
+; CHECK-CVT-GI-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-CVT-GI-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-CVT-GI-NEXT:    fcvtl v2.2d, v1.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v1.2d, v1.4s
+; CHECK-CVT-GI-NEXT:    fcvtl v3.2d, v0.2s
+; CHECK-CVT-GI-NEXT:    fcvtl2 v4.2d, v0.4s
+; CHECK-CVT-GI-NEXT:    fcvtzu v0.2d, v2.2d
+; CHECK-CVT-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-CVT-GI-NEXT:    fcvtzu v2.2d, v3.2d
+; CHECK-CVT-GI-NEXT:    fcvtzu v3.2d, v4.2d
+; CHECK-CVT-GI-NEXT:    ret
+;
+; CHECK-BF16-GI-LABEL: test_fptoui_sat_i64:
+; CHECK-BF16-GI:       // %bb.0:
+; CHECK-BF16-GI-NEXT:    shll v1.4s, v0.4h, #16
+; CHECK-BF16-GI-NEXT:    shll2 v0.4s, v0.8h, #16
+; CHECK-BF16-GI-NEXT:    fcvtl v2.2d, v1.2s
+; CHECK-BF16-GI-NEXT:    fcvtl2 v1.2d, v1.4s
+; CHECK-BF16-GI-NEXT:    fcvtl v3.2d, v0.2s
+; CHECK-BF16-GI-NEXT:    fcvtl2 v4.2d, v0.4s
+; CHECK-BF16-GI-NEXT:    fcvtzu v0.2d, v2.2d
+; CHECK-BF16-GI-NEXT:    fcvtzu v1.2d, v1.2d
+; CHECK-BF16-GI-NEXT:    fcvtzu v2.2d, v3.2d
+; CHECK-BF16-GI-NEXT:    fcvtzu v3.2d, v4.2d
+; CHECK-BF16-GI-NEXT:    ret
   %i = call <8 x i64> @llvm.fptoui.sat.v8i64.v8bf16(<8 x bfloat> %a)
   ret <8 x i64> %i
 }

diff  --git a/llvm/test/CodeGen/AArch64/round-fptosi-sat-scalar.ll b/llvm/test/CodeGen/AArch64/round-fptosi-sat-scalar.ll
index c39960ab2e25a..c1c10435cf8ab 100644
--- a/llvm/test/CodeGen/AArch64/round-fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/AArch64/round-fptosi-sat-scalar.ll
@@ -1,38 +1,81 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT
 ; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-FP16
-; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=2 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; RUN: llc < %s -mtriple aarch64 -global-isel -global-isel-abort=2 -mattr=+fullfp16 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI-FP16
-
-; CHECK-GI-FP16:       warning: Instruction selection used fallback path for testmswbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testmsxbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testpswbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testpsxbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testzswbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testzsxbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testaswbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testasxbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testnswbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testnsxbf
+; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc < %s -mtriple aarch64 -global-isel -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-GI-FP16
 
 ; Round towards minus infinity (fcvtms).
 
 define i32 @testmswbf(bfloat %a) {
-; CHECK-LABEL: testmswbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintm s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzs w0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testmswbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintm s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzs w0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testmswbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintm s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzs w0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testmswbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintm s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzs w0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testmswbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintm s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzs w0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.floor.bf16(bfloat %a)
   %i = call i32 @llvm.fptosi.sat.i32.bf16(bfloat %r)
@@ -40,21 +83,75 @@ entry:
 }
 
 define i64 @testmsxbf(bfloat %a) {
-; CHECK-LABEL: testmsxbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintm s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzs x0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testmsxbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintm s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzs x0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testmsxbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintm s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzs x0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testmsxbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintm s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzs x0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testmsxbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintm s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzs x0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.floor.bf16(bfloat %a)
   %i = call i64 @llvm.fptosi.sat.i64.bf16(bfloat %r)
@@ -176,21 +273,75 @@ entry:
 ; Round towards plus infinity (fcvtps).
 
 define i32 @testpswbf(bfloat %a) {
-; CHECK-LABEL: testpswbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintp s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzs w0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testpswbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintp s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzs w0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testpswbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintp s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzs w0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testpswbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintp s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzs w0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testpswbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintp s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzs w0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.ceil.bf16(bfloat %a)
   %i = call i32 @llvm.fptosi.sat.i32.bf16(bfloat %r)
@@ -198,21 +349,75 @@ entry:
 }
 
 define i64 @testpsxbf(bfloat %a) {
-; CHECK-LABEL: testpsxbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintp s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzs x0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testpsxbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintp s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzs x0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testpsxbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintp s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzs x0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testpsxbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintp s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzs x0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testpsxbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintp s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzs x0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.ceil.bf16(bfloat %a)
   %i = call i64 @llvm.fptosi.sat.i64.bf16(bfloat %r)
@@ -334,21 +539,75 @@ entry:
 ; Round towards zero (fcvtzs).
 
 define i32 @testzswbf(bfloat %a) {
-; CHECK-LABEL: testzswbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintz s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzs w0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testzswbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintz s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzs w0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testzswbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintz s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzs w0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testzswbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintz s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzs w0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testzswbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintz s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzs w0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.trunc.bf16(bfloat %a)
   %i = call i32 @llvm.fptosi.sat.i32.bf16(bfloat %r)
@@ -356,21 +615,75 @@ entry:
 }
 
 define i64 @testzsxbf(bfloat %a) {
-; CHECK-LABEL: testzsxbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintz s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzs x0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testzsxbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintz s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzs x0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testzsxbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintz s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzs x0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testzsxbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintz s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzs x0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testzsxbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintz s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzs x0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.trunc.bf16(bfloat %a)
   %i = call i64 @llvm.fptosi.sat.i64.bf16(bfloat %r)
@@ -492,21 +805,75 @@ entry:
 ; Round to nearest, ties away from zero (fcvtas).
 
 define i32 @testaswbf(bfloat %a) {
-; CHECK-LABEL: testaswbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frinta s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzs w0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testaswbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frinta s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzs w0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testaswbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frinta s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzs w0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testaswbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frinta s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzs w0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testaswbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frinta s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzs w0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.round.bf16(bfloat %a)
   %i = call i32 @llvm.fptosi.sat.i32.bf16(bfloat %r)
@@ -514,21 +881,75 @@ entry:
 }
 
 define i64 @testasxbf(bfloat %a) {
-; CHECK-LABEL: testasxbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frinta s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzs x0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testasxbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frinta s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzs x0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testasxbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frinta s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzs x0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testasxbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frinta s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzs x0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testasxbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frinta s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzs x0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.round.bf16(bfloat %a)
   %i = call i64 @llvm.fptosi.sat.i64.bf16(bfloat %r)
@@ -650,21 +1071,75 @@ entry:
 ; Round to nearest, ties to even (fcvtns).
 
 define i32 @testnswbf(bfloat %a) {
-; CHECK-LABEL: testnswbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintn s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzs w0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testnswbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintn s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzs w0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testnswbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintn s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzs w0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testnswbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintn s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzs w0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testnswbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintn s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzs w0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.roundeven.bf16(bfloat %a)
   %i = call i32 @llvm.fptosi.sat.i32.bf16(bfloat %r)
@@ -672,21 +1147,75 @@ entry:
 }
 
 define i64 @testnsxbf(bfloat %a) {
-; CHECK-LABEL: testnsxbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintn s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzs x0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testnsxbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintn s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzs x0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testnsxbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintn s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzs x0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testnsxbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintn s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzs x0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testnsxbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintn s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzs x0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.roundeven.bf16(bfloat %a)
   %i = call i64 @llvm.fptosi.sat.i64.bf16(bfloat %r)

diff  --git a/llvm/test/CodeGen/AArch64/round-fptoui-sat-scalar.ll b/llvm/test/CodeGen/AArch64/round-fptoui-sat-scalar.ll
index ea2dd52ce958a..9d449976b72a9 100644
--- a/llvm/test/CodeGen/AArch64/round-fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/AArch64/round-fptoui-sat-scalar.ll
@@ -1,38 +1,82 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
 ; RUN: llc < %s -mtriple=aarch64 | FileCheck %s --check-prefixes=CHECK,CHECK-CVT
 ; RUN: llc < %s -mtriple=aarch64 -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-FP16
-; RUN: llc < %s -mtriple=aarch64 -global-isel -global-isel-abort=2 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-; RUN: llc < %s -mtriple aarch64 -global-isel -global-isel-abort=2 -mattr=+fullfp16 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI-FP16
-
-; CHECK-GI-FP16:       warning: Instruction selection used fallback path for testmuwbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testmuxbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testpuwbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testpuxbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testzuwbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testzuxbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testauwbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testauxbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testnuwbf
-; CHECK-GI-FP16-NEXT:  warning: Instruction selection used fallback path for testnuxbf
+; RUN: llc < %s -mtriple=aarch64 -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+; RUN: llc < %s -mtriple aarch64 -global-isel -mattr=+fullfp16 | FileCheck %s --check-prefixes=CHECK,CHECK-GI-FP16
+
 
 ; Round towards minus infinity (fcvtmu).
 
 define i32 @testmuwbf(bfloat %a) {
-; CHECK-LABEL: testmuwbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintm s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzu w0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testmuwbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintm s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzu w0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testmuwbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintm s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzu w0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testmuwbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintm s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzu w0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testmuwbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintm s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzu w0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.floor.bf16(bfloat %a)
   %i = call i32 @llvm.fptoui.sat.i32.bf16(bfloat %r)
@@ -40,21 +84,75 @@ entry:
 }
 
 define i64 @testmuxbf(bfloat %a) {
-; CHECK-LABEL: testmuxbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintm s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzu x0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testmuxbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintm s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzu x0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testmuxbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintm s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzu x0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testmuxbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintm s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzu x0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testmuxbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintm s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzu x0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.floor.bf16(bfloat %a)
   %i = call i64 @llvm.fptoui.sat.i64.bf16(bfloat %r)
@@ -176,21 +274,75 @@ entry:
 ; Round towards plus infinity (fcvtpu).
 
 define i32 @testpuwbf(bfloat %a) {
-; CHECK-LABEL: testpuwbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintp s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzu w0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testpuwbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintp s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzu w0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testpuwbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintp s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzu w0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testpuwbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintp s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzu w0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testpuwbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintp s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzu w0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.ceil.bf16(bfloat %a)
   %i = call i32 @llvm.fptoui.sat.i32.bf16(bfloat %r)
@@ -198,21 +350,75 @@ entry:
 }
 
 define i64 @testpuxbf(bfloat %a) {
-; CHECK-LABEL: testpuxbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintp s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzu x0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testpuxbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintp s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzu x0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testpuxbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintp s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzu x0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testpuxbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintp s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzu x0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testpuxbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintp s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzu x0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.ceil.bf16(bfloat %a)
   %i = call i64 @llvm.fptoui.sat.i64.bf16(bfloat %r)
@@ -334,21 +540,75 @@ entry:
 ; Round towards zero (fcvtzu).
 
 define i32 @testzuwbf(bfloat %a) {
-; CHECK-LABEL: testzuwbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintz s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzu w0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testzuwbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintz s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzu w0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testzuwbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintz s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzu w0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testzuwbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintz s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzu w0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testzuwbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintz s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzu w0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.trunc.bf16(bfloat %a)
   %i = call i32 @llvm.fptoui.sat.i32.bf16(bfloat %r)
@@ -356,21 +616,75 @@ entry:
 }
 
 define i64 @testzuxbf(bfloat %a) {
-; CHECK-LABEL: testzuxbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintz s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzu x0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testzuxbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintz s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzu x0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testzuxbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintz s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzu x0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testzuxbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintz s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzu x0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testzuxbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintz s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzu x0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.trunc.bf16(bfloat %a)
   %i = call i64 @llvm.fptoui.sat.i64.bf16(bfloat %r)
@@ -492,21 +806,75 @@ entry:
 ; Round to nearest, ties away from zero (fcvtau).
 
 define i32 @testauwbf(bfloat %a) {
-; CHECK-LABEL: testauwbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frinta s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzu w0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testauwbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frinta s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzu w0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testauwbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frinta s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzu w0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testauwbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frinta s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzu w0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testauwbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frinta s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzu w0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.round.bf16(bfloat %a)
   %i = call i32 @llvm.fptoui.sat.i32.bf16(bfloat %r)
@@ -514,21 +882,75 @@ entry:
 }
 
 define i64 @testauxbf(bfloat %a) {
-; CHECK-LABEL: testauxbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frinta s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzu x0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testauxbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frinta s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzu x0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testauxbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frinta s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzu x0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testauxbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frinta s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzu x0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testauxbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frinta s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzu x0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.round.bf16(bfloat %a)
   %i = call i64 @llvm.fptoui.sat.i64.bf16(bfloat %r)
@@ -650,21 +1072,75 @@ entry:
 ; Round to nearest, ties to even (fcvtnu).
 
 define i32 @testnuwbf(bfloat %a) {
-; CHECK-LABEL: testnuwbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintn s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzu w0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testnuwbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintn s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzu w0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testnuwbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintn s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzu w0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testnuwbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintn s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzu w0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testnuwbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintn s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzu w0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.roundeven.bf16(bfloat %a)
   %i = call i32 @llvm.fptoui.sat.i32.bf16(bfloat %r)
@@ -672,21 +1148,75 @@ entry:
 }
 
 define i64 @testnuxbf(bfloat %a) {
-; CHECK-LABEL: testnuxbf:
-; CHECK:       // %bb.0: // %entry
-; CHECK-NEXT:    // kill: def $h0 killed $h0 def $d0
-; CHECK-NEXT:    mov w8, #32767 // =0x7fff
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    frintn s0, s0
-; CHECK-NEXT:    fmov w9, s0
-; CHECK-NEXT:    ubfx w10, w9, #16, #1
-; CHECK-NEXT:    add w8, w9, w8
-; CHECK-NEXT:    add w8, w10, w8
-; CHECK-NEXT:    lsr w8, w8, #16
-; CHECK-NEXT:    fmov s0, w8
-; CHECK-NEXT:    shll v0.4s, v0.4h, #16
-; CHECK-NEXT:    fcvtzu x0, s0
-; CHECK-NEXT:    ret
+; CHECK-CVT-LABEL: testnuxbf:
+; CHECK-CVT:       // %bb.0: // %entry
+; CHECK-CVT-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-CVT-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    frintn s0, s0
+; CHECK-CVT-NEXT:    fmov w9, s0
+; CHECK-CVT-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-CVT-NEXT:    add w8, w9, w8
+; CHECK-CVT-NEXT:    add w8, w10, w8
+; CHECK-CVT-NEXT:    lsr w8, w8, #16
+; CHECK-CVT-NEXT:    fmov s0, w8
+; CHECK-CVT-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-CVT-NEXT:    fcvtzu x0, s0
+; CHECK-CVT-NEXT:    ret
+;
+; CHECK-FP16-LABEL: testnuxbf:
+; CHECK-FP16:       // %bb.0: // %entry
+; CHECK-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    frintn s0, s0
+; CHECK-FP16-NEXT:    fmov w9, s0
+; CHECK-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-FP16-NEXT:    add w8, w9, w8
+; CHECK-FP16-NEXT:    add w8, w10, w8
+; CHECK-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-FP16-NEXT:    fmov s0, w8
+; CHECK-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-FP16-NEXT:    fcvtzu x0, s0
+; CHECK-FP16-NEXT:    ret
+;
+; CHECK-GI-LABEL: testnuxbf:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    frintn s0, s0
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fcmp s0, #0.0
+; CHECK-GI-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-NEXT:    add w8, w9, w8
+; CHECK-GI-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-NEXT:    add w8, w8, w10
+; CHECK-GI-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-NEXT:    lsr w8, w8, #16
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-NEXT:    fcvtzu x0, s0
+; CHECK-GI-NEXT:    ret
+;
+; CHECK-GI-FP16-LABEL: testnuxbf:
+; CHECK-GI-FP16:       // %bb.0: // %entry
+; CHECK-GI-FP16-NEXT:    // kill: def $h0 killed $h0 def $d0
+; CHECK-GI-FP16-NEXT:    mov w8, #32767 // =0x7fff
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    frintn s0, s0
+; CHECK-GI-FP16-NEXT:    fmov w9, s0
+; CHECK-GI-FP16-NEXT:    fcmp s0, #0.0
+; CHECK-GI-FP16-NEXT:    ubfx w10, w9, #16, #1
+; CHECK-GI-FP16-NEXT:    add w8, w9, w8
+; CHECK-GI-FP16-NEXT:    orr w9, w9, #0x400000
+; CHECK-GI-FP16-NEXT:    add w8, w8, w10
+; CHECK-GI-FP16-NEXT:    csel w8, w9, w8, vs
+; CHECK-GI-FP16-NEXT:    lsr w8, w8, #16
+; CHECK-GI-FP16-NEXT:    fmov s0, w8
+; CHECK-GI-FP16-NEXT:    shll v0.4s, v0.4h, #16
+; CHECK-GI-FP16-NEXT:    fcvtzu x0, s0
+; CHECK-GI-FP16-NEXT:    ret
 entry:
   %r = call bfloat @llvm.roundeven.bf16(bfloat %a)
   %i = call i64 @llvm.fptoui.sat.i64.bf16(bfloat %r)