[llvm-branch-commits] [llvm] release/23.x: [AArch64] fix invalid GISel lowering of saturating `f16` to integer conversion (#235676) (PR #240155)

Folkert de Vries via llvm-branch-commits llvm-branch-commits at lists.llvm.org
Sun Nov 8 07:41:08 PST 2026


https://github.com/folkertdev created https://github.com/llvm/llvm-project/pull/240155

Backport 617b7da9db855f1c74e9425555ce23f98f4dc6f9

Requested by: @folkertdev

This is a custom PR because the automatic https://github.com/llvm/llvm-project/pull/238543 ran into test failures. I've updated the CHECKs here.

>From a8ec91434f566eef428682aa52e6de2c72f7e130 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Tue, 3 Nov 2026 14:58:05 +0100
Subject: [PATCH 1/2] [AArch64] fix invalid GISel lowering of saturating `f16`
 to integer conversion (#235676)

The saturating version of the code seems to be copied from the
non-saturating version, and has this comment

> The range of a fp16 value fits into an i17, so we can lower the width
to i64.

But that is wrong for the saturating conversions, because positive
infinity is mapped to `i128::MAX`.

There may be a way to recover some of the shorter codegen, but I'm
hoping this change is something we can easily backport.

cc https://github.com/rust-lang/rust/issues/164972

(cherry picked from commit 617b7da9db855f1c74e9425555ce23f98f4dc6f9)
---
 .../AArch64/GISel/AArch64LegalizerInfo.cpp    |   9 +-
 .../test/CodeGen/AArch64/fptosi-sat-scalar.ll |  76 ++-
 .../test/CodeGen/AArch64/fptosi-sat-vector.ll | 574 ++++++++++++++----
 .../test/CodeGen/AArch64/fptoui-sat-scalar.ll |  42 +-
 .../test/CodeGen/AArch64/fptoui-sat-vector.ll | 394 ++++++++----
 5 files changed, 837 insertions(+), 258 deletions(-)

diff --git a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
index 387ed79893d3c6..b076f384d69207 100644
--- a/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
+++ b/llvm/lib/Target/AArch64/GISel/AArch64LegalizerInfo.cpp
@@ -974,13 +974,8 @@ AArch64LegalizerInfo::AArch64LegalizerInfo(const AArch64Subtarget &ST)
       // Handle types larger than i64 by scalarizing/lowering.
       .scalarizeIf(scalarOrEltWiderThan(0, 64), 0)
       .scalarizeIf(scalarOrEltWiderThan(1, 64), 1)
-      // The range of a fp16 value fits into an i17, so we can lower the width
-      // to i64.
-      .narrowScalarIf(
-          [=](const LegalityQuery &Query) {
-            return Query.Types[1] == f16 && Query.Types[0].getSizeInBits() > 64;
-          },
-          changeTo(0, i64))
+      // NOTE: narrowScalarIf is invalid for the saturating conversions,
+      // infinities need the full width of the target integer type.
       .lowerIf(::any(scalarWiderThan(0, 64), scalarWiderThan(1, 64)), 0)
       .moreElementsToNextPow2(0)
       .widenScalarToNextPow2(0, /*MinSize=*/32)
diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll
index f95d338198c4d1..d2376885f94de5 100644
--- a/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll
@@ -877,14 +877,44 @@ define i100 @test_signed_i100_f16(half %f) nounwind {
 ; CHECK-GI-CVT-LABEL: test_signed_i100_f16:
 ; CHECK-GI-CVT:       // %bb.0:
 ; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    mov x1, xzr
-; CHECK-GI-CVT-NEXT:    fcvtzs x0, s0
+; CHECK-GI-CVT-NEXT:    mov w9, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    mov w10, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    movk w9, #51071, lsl #16
+; CHECK-GI-CVT-NEXT:    movk w10, #18303, lsl #16
+; CHECK-GI-CVT-NEXT:    fmov s1, w9
+; CHECK-GI-CVT-NEXT:    fcvtzs x8, s0
+; CHECK-GI-CVT-NEXT:    fcmp s0, s1
+; CHECK-GI-CVT-NEXT:    fmov s1, w10
+; CHECK-GI-CVT-NEXT:    mov x10, #34359738368 // =0x800000000
+; CHECK-GI-CVT-NEXT:    asr x9, x8, #63
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-CVT-NEXT:    csel x9, x10, x9, lt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s1
+; CHECK-GI-CVT-NEXT:    mov x10, #34359738367 // =0x7ffffffff
+; CHECK-GI-CVT-NEXT:    csinv x8, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x9, x10, x9, gt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s0
+; CHECK-GI-CVT-NEXT:    csel x0, xzr, x8, vs
+; CHECK-GI-CVT-NEXT:    csel x1, xzr, x9, vs
 ; CHECK-GI-CVT-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: test_signed_i100_f16:
 ; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzs x0, h0
-; CHECK-GI-FP16-NEXT:    mov x1, xzr
+; CHECK-GI-FP16-NEXT:    mvni v1.4h, #4, lsl #8
+; CHECK-GI-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-GI-FP16-NEXT:    mov x10, #34359738368 // =0x800000000
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    asr x9, x8, #63
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-FP16-NEXT:    csel x9, x10, x9, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    mov x10, #34359738367 // =0x7ffffffff
+; CHECK-GI-FP16-NEXT:    csinv x8, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x9, x10, x9, gt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h0
+; CHECK-GI-FP16-NEXT:    csel x0, xzr, x8, vs
+; CHECK-GI-FP16-NEXT:    csel x1, xzr, x9, vs
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call i100 @llvm.fptosi.sat.i100.f16(half %f)
     ret i100 %x
@@ -919,14 +949,44 @@ define i128 @test_signed_i128_f16(half %f) nounwind {
 ; CHECK-GI-CVT-LABEL: test_signed_i128_f16:
 ; CHECK-GI-CVT:       // %bb.0:
 ; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    mov x1, xzr
-; CHECK-GI-CVT-NEXT:    fcvtzs x0, s0
+; CHECK-GI-CVT-NEXT:    mov w9, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    mov w10, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    movk w9, #51071, lsl #16
+; CHECK-GI-CVT-NEXT:    movk w10, #18303, lsl #16
+; CHECK-GI-CVT-NEXT:    fmov s1, w9
+; CHECK-GI-CVT-NEXT:    fcvtzs x8, s0
+; CHECK-GI-CVT-NEXT:    fcmp s0, s1
+; CHECK-GI-CVT-NEXT:    fmov s1, w10
+; CHECK-GI-CVT-NEXT:    mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-CVT-NEXT:    asr x9, x8, #63
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-CVT-NEXT:    csel x9, x10, x9, lt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s1
+; CHECK-GI-CVT-NEXT:    mov x10, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-CVT-NEXT:    csinv x8, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x9, x10, x9, gt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s0
+; CHECK-GI-CVT-NEXT:    csel x0, xzr, x8, vs
+; CHECK-GI-CVT-NEXT:    csel x1, xzr, x9, vs
 ; CHECK-GI-CVT-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: test_signed_i128_f16:
 ; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzs x0, h0
-; CHECK-GI-FP16-NEXT:    mov x1, xzr
+; CHECK-GI-FP16-NEXT:    mvni v1.4h, #4, lsl #8
+; CHECK-GI-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-GI-FP16-NEXT:    mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    asr x9, x8, #63
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-FP16-NEXT:    csel x9, x10, x9, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    mov x10, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-FP16-NEXT:    csinv x8, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x9, x10, x9, gt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h0
+; CHECK-GI-FP16-NEXT:    csel x0, xzr, x8, vs
+; CHECK-GI-FP16-NEXT:    csel x1, xzr, x9, vs
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call i128 @llvm.fptosi.sat.i128.f16(half %f)
     ret i128 %x
diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
index 145aef9123a4e9..b756cc31f57a5c 100644
--- a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
@@ -3237,37 +3237,121 @@ define <4 x i100> @test_signed_v4f16_v4i100(<4 x half> %f) {
 ; CHECK-GI-CVT-LABEL: test_signed_v4f16_v4i100:
 ; CHECK-GI-CVT:       // %bb.0:
 ; CHECK-GI-CVT-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-CVT-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-CVT-NEXT:    mov x1, xzr
-; CHECK-GI-CVT-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    mov x3, xzr
-; CHECK-GI-CVT-NEXT:    mov x5, xzr
-; CHECK-GI-CVT-NEXT:    mov x7, xzr
-; CHECK-GI-CVT-NEXT:    fcvt s1, h1
-; CHECK-GI-CVT-NEXT:    fcvt s2, h2
+; CHECK-GI-CVT-NEXT:    fcvt s2, h0
+; CHECK-GI-CVT-NEXT:    mov w9, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-CVT-NEXT:    movk w9, #51071, lsl #16
+; CHECK-GI-CVT-NEXT:    mov x11, #34359738368 // =0x800000000
+; CHECK-GI-CVT-NEXT:    mov h5, v0.h[2]
+; CHECK-GI-CVT-NEXT:    fmov s1, w9
+; CHECK-GI-CVT-NEXT:    mov w9, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-CVT-NEXT:    movk w9, #18303, lsl #16
+; CHECK-GI-CVT-NEXT:    fcvtzs x8, s2
+; CHECK-GI-CVT-NEXT:    fmov s4, w9
 ; CHECK-GI-CVT-NEXT:    fcvt s3, h3
-; CHECK-GI-CVT-NEXT:    fcvtzs x0, s0
-; CHECK-GI-CVT-NEXT:    fcvtzs x2, s1
-; CHECK-GI-CVT-NEXT:    fcvtzs x4, s2
-; CHECK-GI-CVT-NEXT:    fcvtzs x6, s3
+; CHECK-GI-CVT-NEXT:    fcmp s2, s1
+; CHECK-GI-CVT-NEXT:    fcvt s0, h0
+; CHECK-GI-CVT-NEXT:    asr x10, x8, #63
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-CVT-NEXT:    fcvtzs x12, s3
+; CHECK-GI-CVT-NEXT:    csel x9, x11, x10, lt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s4
+; CHECK-GI-CVT-NEXT:    mov x10, #34359738367 // =0x7ffffffff
+; CHECK-GI-CVT-NEXT:    csinv x8, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x9, x10, x9, gt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s2
+; CHECK-GI-CVT-NEXT:    fcvt s2, h5
+; CHECK-GI-CVT-NEXT:    csel x0, xzr, x8, vs
+; CHECK-GI-CVT-NEXT:    csel x1, xzr, x9, vs
+; CHECK-GI-CVT-NEXT:    fcmp s3, s1
+; CHECK-GI-CVT-NEXT:    asr x8, x12, #63
+; CHECK-GI-CVT-NEXT:    csel x9, xzr, x12, lt
+; CHECK-GI-CVT-NEXT:    csel x8, x11, x8, lt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s4
+; CHECK-GI-CVT-NEXT:    fcvtzs x12, s2
+; CHECK-GI-CVT-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x8, x10, x8, gt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s3
+; CHECK-GI-CVT-NEXT:    csel x2, xzr, x9, vs
+; CHECK-GI-CVT-NEXT:    csel x3, xzr, x8, vs
+; CHECK-GI-CVT-NEXT:    fcmp s2, s1
+; CHECK-GI-CVT-NEXT:    asr x8, x12, #63
+; CHECK-GI-CVT-NEXT:    csel x9, xzr, x12, lt
+; CHECK-GI-CVT-NEXT:    csel x8, x11, x8, lt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s4
+; CHECK-GI-CVT-NEXT:    fcvtzs x12, s0
+; CHECK-GI-CVT-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x8, x10, x8, gt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s2
+; CHECK-GI-CVT-NEXT:    csel x4, xzr, x9, vs
+; CHECK-GI-CVT-NEXT:    csel x5, xzr, x8, vs
+; CHECK-GI-CVT-NEXT:    fcmp s0, s1
+; CHECK-GI-CVT-NEXT:    asr x8, x12, #63
+; CHECK-GI-CVT-NEXT:    csel x9, xzr, x12, lt
+; CHECK-GI-CVT-NEXT:    csel x8, x11, x8, lt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s4
+; CHECK-GI-CVT-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x8, x10, x8, gt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s0
+; CHECK-GI-CVT-NEXT:    csel x6, xzr, x9, vs
+; CHECK-GI-CVT-NEXT:    csel x7, xzr, x8, vs
 ; CHECK-GI-CVT-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: test_signed_v4f16_v4i100:
 ; CHECK-GI-FP16:       // %bb.0:
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov x1, xzr
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT:    fcvtzs x0, h0
-; CHECK-GI-FP16-NEXT:    mov x3, xzr
-; CHECK-GI-FP16-NEXT:    mov x5, xzr
-; CHECK-GI-FP16-NEXT:    mov x7, xzr
-; CHECK-GI-FP16-NEXT:    fcvtzs x2, h1
-; CHECK-GI-FP16-NEXT:    fcvtzs x4, h2
-; CHECK-GI-FP16-NEXT:    fcvtzs x6, h3
+; CHECK-GI-FP16-NEXT:    mvni v1.4h, #4, lsl #8
+; CHECK-GI-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-GI-FP16-NEXT:    mov x10, #34359738368 // =0x800000000
+; CHECK-GI-FP16-NEXT:    mvni v2.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov x11, #34359738367 // =0x7ffffffff
+; CHECK-GI-FP16-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    asr x9, x8, #63
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h3
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-FP16-NEXT:    csel x9, x10, x9, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h2
+; CHECK-GI-FP16-NEXT:    csinv x8, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x9, x11, x9, gt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h0
+; CHECK-GI-FP16-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-FP16-NEXT:    csel x0, xzr, x8, vs
+; CHECK-GI-FP16-NEXT:    csel x1, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    asr x8, x12, #63
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x12, lt
+; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h2
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h4
+; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h3
+; CHECK-GI-FP16-NEXT:    csel x2, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    csel x3, xzr, x8, vs
+; CHECK-GI-FP16-NEXT:    fcmp h4, h1
+; CHECK-GI-FP16-NEXT:    asr x8, x12, #63
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x12, lt
+; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
+; CHECK-GI-FP16-NEXT:    fcmp h4, h2
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h0
+; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
+; CHECK-GI-FP16-NEXT:    fcmp h4, h4
+; CHECK-GI-FP16-NEXT:    csel x4, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    csel x5, xzr, x8, vs
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    asr x8, x12, #63
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x12, lt
+; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h2
+; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h0
+; CHECK-GI-FP16-NEXT:    csel x6, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    csel x7, xzr, x8, vs
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call <4 x i100> @llvm.fptosi.sat.v4f16.v4i100(<4 x half> %f)
     ret <4 x i100> %x
@@ -3377,37 +3461,121 @@ define <4 x i128> @test_signed_v4f16_v4i128(<4 x half> %f) {
 ; CHECK-GI-CVT-LABEL: test_signed_v4f16_v4i128:
 ; CHECK-GI-CVT:       // %bb.0:
 ; CHECK-GI-CVT-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-CVT-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-CVT-NEXT:    mov x1, xzr
-; CHECK-GI-CVT-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    mov x3, xzr
-; CHECK-GI-CVT-NEXT:    mov x5, xzr
-; CHECK-GI-CVT-NEXT:    mov x7, xzr
-; CHECK-GI-CVT-NEXT:    fcvt s1, h1
-; CHECK-GI-CVT-NEXT:    fcvt s2, h2
+; CHECK-GI-CVT-NEXT:    fcvt s2, h0
+; CHECK-GI-CVT-NEXT:    mov w9, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-CVT-NEXT:    movk w9, #51071, lsl #16
+; CHECK-GI-CVT-NEXT:    mov x11, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-CVT-NEXT:    mov h5, v0.h[2]
+; CHECK-GI-CVT-NEXT:    fmov s1, w9
+; CHECK-GI-CVT-NEXT:    mov w9, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-CVT-NEXT:    movk w9, #18303, lsl #16
+; CHECK-GI-CVT-NEXT:    fcvtzs x8, s2
+; CHECK-GI-CVT-NEXT:    fmov s4, w9
 ; CHECK-GI-CVT-NEXT:    fcvt s3, h3
-; CHECK-GI-CVT-NEXT:    fcvtzs x0, s0
-; CHECK-GI-CVT-NEXT:    fcvtzs x2, s1
-; CHECK-GI-CVT-NEXT:    fcvtzs x4, s2
-; CHECK-GI-CVT-NEXT:    fcvtzs x6, s3
+; CHECK-GI-CVT-NEXT:    fcmp s2, s1
+; CHECK-GI-CVT-NEXT:    fcvt s0, h0
+; CHECK-GI-CVT-NEXT:    asr x10, x8, #63
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-CVT-NEXT:    fcvtzs x12, s3
+; CHECK-GI-CVT-NEXT:    csel x9, x11, x10, lt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s4
+; CHECK-GI-CVT-NEXT:    mov x10, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-CVT-NEXT:    csinv x8, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x9, x10, x9, gt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s2
+; CHECK-GI-CVT-NEXT:    fcvt s2, h5
+; CHECK-GI-CVT-NEXT:    csel x0, xzr, x8, vs
+; CHECK-GI-CVT-NEXT:    csel x1, xzr, x9, vs
+; CHECK-GI-CVT-NEXT:    fcmp s3, s1
+; CHECK-GI-CVT-NEXT:    asr x8, x12, #63
+; CHECK-GI-CVT-NEXT:    csel x9, xzr, x12, lt
+; CHECK-GI-CVT-NEXT:    csel x8, x11, x8, lt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s4
+; CHECK-GI-CVT-NEXT:    fcvtzs x12, s2
+; CHECK-GI-CVT-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x8, x10, x8, gt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s3
+; CHECK-GI-CVT-NEXT:    csel x2, xzr, x9, vs
+; CHECK-GI-CVT-NEXT:    csel x3, xzr, x8, vs
+; CHECK-GI-CVT-NEXT:    fcmp s2, s1
+; CHECK-GI-CVT-NEXT:    asr x8, x12, #63
+; CHECK-GI-CVT-NEXT:    csel x9, xzr, x12, lt
+; CHECK-GI-CVT-NEXT:    csel x8, x11, x8, lt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s4
+; CHECK-GI-CVT-NEXT:    fcvtzs x12, s0
+; CHECK-GI-CVT-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x8, x10, x8, gt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s2
+; CHECK-GI-CVT-NEXT:    csel x4, xzr, x9, vs
+; CHECK-GI-CVT-NEXT:    csel x5, xzr, x8, vs
+; CHECK-GI-CVT-NEXT:    fcmp s0, s1
+; CHECK-GI-CVT-NEXT:    asr x8, x12, #63
+; CHECK-GI-CVT-NEXT:    csel x9, xzr, x12, lt
+; CHECK-GI-CVT-NEXT:    csel x8, x11, x8, lt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s4
+; CHECK-GI-CVT-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x8, x10, x8, gt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s0
+; CHECK-GI-CVT-NEXT:    csel x6, xzr, x9, vs
+; CHECK-GI-CVT-NEXT:    csel x7, xzr, x8, vs
 ; CHECK-GI-CVT-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: test_signed_v4f16_v4i128:
 ; CHECK-GI-FP16:       // %bb.0:
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov x1, xzr
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT:    fcvtzs x0, h0
-; CHECK-GI-FP16-NEXT:    mov x3, xzr
-; CHECK-GI-FP16-NEXT:    mov x5, xzr
-; CHECK-GI-FP16-NEXT:    mov x7, xzr
-; CHECK-GI-FP16-NEXT:    fcvtzs x2, h1
-; CHECK-GI-FP16-NEXT:    fcvtzs x4, h2
-; CHECK-GI-FP16-NEXT:    fcvtzs x6, h3
+; CHECK-GI-FP16-NEXT:    mvni v1.4h, #4, lsl #8
+; CHECK-GI-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-GI-FP16-NEXT:    mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-FP16-NEXT:    mvni v2.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov x11, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-FP16-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    asr x9, x8, #63
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h3
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-FP16-NEXT:    csel x9, x10, x9, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h2
+; CHECK-GI-FP16-NEXT:    csinv x8, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x9, x11, x9, gt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h0
+; CHECK-GI-FP16-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-FP16-NEXT:    csel x0, xzr, x8, vs
+; CHECK-GI-FP16-NEXT:    csel x1, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    asr x8, x12, #63
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x12, lt
+; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h2
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h4
+; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h3
+; CHECK-GI-FP16-NEXT:    csel x2, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    csel x3, xzr, x8, vs
+; CHECK-GI-FP16-NEXT:    fcmp h4, h1
+; CHECK-GI-FP16-NEXT:    asr x8, x12, #63
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x12, lt
+; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
+; CHECK-GI-FP16-NEXT:    fcmp h4, h2
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h0
+; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
+; CHECK-GI-FP16-NEXT:    fcmp h4, h4
+; CHECK-GI-FP16-NEXT:    csel x4, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    csel x5, xzr, x8, vs
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    asr x8, x12, #63
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x12, lt
+; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h2
+; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h0
+; CHECK-GI-FP16-NEXT:    csel x6, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    csel x7, xzr, x8, vs
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call <4 x i128> @llvm.fptosi.sat.v4f16.v4i128(<4 x half> %f)
     ret <4 x i128> %x
@@ -4209,86 +4377,258 @@ define <8 x i128> @test_signed_v8f16_v8i128(<8 x half> %f) {
 ;
 ; CHECK-GI-CVT-LABEL: test_signed_v8f16_v8i128:
 ; CHECK-GI-CVT:       // %bb.0:
-; CHECK-GI-CVT-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-CVT-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-CVT-NEXT:    mov h4, v0.h[4]
-; CHECK-GI-CVT-NEXT:    fcvt s5, h0
-; CHECK-GI-CVT-NEXT:    mov h6, v0.h[5]
-; CHECK-GI-CVT-NEXT:    mov h7, v0.h[6]
+; CHECK-GI-CVT-NEXT:    fcvt s3, h0
+; CHECK-GI-CVT-NEXT:    mov w10, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-CVT-NEXT:    movk w10, #51071, lsl #16
+; CHECK-GI-CVT-NEXT:    mov w11, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    mov h5, v0.h[2]
+; CHECK-GI-CVT-NEXT:    fmov s1, w10
+; CHECK-GI-CVT-NEXT:    movk w11, #18303, lsl #16
+; CHECK-GI-CVT-NEXT:    mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-CVT-NEXT:    fcvtzs x9, s3
+; CHECK-GI-CVT-NEXT:    fcvt s4, h2
+; CHECK-GI-CVT-NEXT:    fmov s2, w11
+; CHECK-GI-CVT-NEXT:    fcmp s3, s1
+; CHECK-GI-CVT-NEXT:    asr x12, x9, #63
+; CHECK-GI-CVT-NEXT:    csel x11, xzr, x9, lt
+; CHECK-GI-CVT-NEXT:    mov x9, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-CVT-NEXT:    fcvtzs x13, s4
+; CHECK-GI-CVT-NEXT:    csel x12, x10, x12, lt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s2
+; CHECK-GI-CVT-NEXT:    csinv x11, x11, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x14, x9, x12, gt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s3
+; CHECK-GI-CVT-NEXT:    fcvt s3, h5
+; CHECK-GI-CVT-NEXT:    mov h5, v0.h[3]
+; CHECK-GI-CVT-NEXT:    csel x12, xzr, x11, vs
+; CHECK-GI-CVT-NEXT:    csel x11, xzr, x14, vs
+; CHECK-GI-CVT-NEXT:    fcmp s4, s1
+; CHECK-GI-CVT-NEXT:    asr x14, x13, #63
+; CHECK-GI-CVT-NEXT:    fcvtzs x15, s3
+; CHECK-GI-CVT-NEXT:    csel x13, xzr, x13, lt
+; CHECK-GI-CVT-NEXT:    csel x14, x10, x14, lt
+; CHECK-GI-CVT-NEXT:    fcmp s4, s2
+; CHECK-GI-CVT-NEXT:    csinv x13, x13, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x16, x9, x14, gt
+; CHECK-GI-CVT-NEXT:    fcmp s4, s4
+; CHECK-GI-CVT-NEXT:    fcvt s4, h5
+; CHECK-GI-CVT-NEXT:    mov h5, v0.h[4]
+; CHECK-GI-CVT-NEXT:    csel x14, xzr, x13, vs
+; CHECK-GI-CVT-NEXT:    csel x13, xzr, x16, vs
+; CHECK-GI-CVT-NEXT:    fcmp s3, s1
+; CHECK-GI-CVT-NEXT:    asr x16, x15, #63
+; CHECK-GI-CVT-NEXT:    fcvtzs x17, s4
+; CHECK-GI-CVT-NEXT:    csel x15, xzr, x15, lt
+; CHECK-GI-CVT-NEXT:    csel x16, x10, x16, lt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s2
+; CHECK-GI-CVT-NEXT:    csinv x15, x15, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x18, x9, x16, gt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s3
+; CHECK-GI-CVT-NEXT:    fcvt s3, h5
+; CHECK-GI-CVT-NEXT:    mov h5, v0.h[5]
+; CHECK-GI-CVT-NEXT:    csel x16, xzr, x15, vs
+; CHECK-GI-CVT-NEXT:    csel x15, xzr, x18, vs
+; CHECK-GI-CVT-NEXT:    fcmp s4, s1
+; CHECK-GI-CVT-NEXT:    asr x18, x17, #63
+; CHECK-GI-CVT-NEXT:    fcvtzs x0, s3
+; CHECK-GI-CVT-NEXT:    csel x17, xzr, x17, lt
+; CHECK-GI-CVT-NEXT:    csel x18, x10, x18, lt
+; CHECK-GI-CVT-NEXT:    fcmp s4, s2
+; CHECK-GI-CVT-NEXT:    csinv x17, x17, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x1, x9, x18, gt
+; CHECK-GI-CVT-NEXT:    fcmp s4, s4
+; CHECK-GI-CVT-NEXT:    fcvt s4, h5
+; CHECK-GI-CVT-NEXT:    mov h5, v0.h[6]
 ; CHECK-GI-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-GI-CVT-NEXT:    fcvt s1, h1
-; CHECK-GI-CVT-NEXT:    fcvt s2, h2
-; CHECK-GI-CVT-NEXT:    fcvt s3, h3
-; CHECK-GI-CVT-NEXT:    fcvtzs x9, s5
-; CHECK-GI-CVT-NEXT:    fcvt s4, h4
-; CHECK-GI-CVT-NEXT:    fcvt s5, h6
+; CHECK-GI-CVT-NEXT:    csel x18, xzr, x17, vs
+; CHECK-GI-CVT-NEXT:    csel x17, xzr, x1, vs
+; CHECK-GI-CVT-NEXT:    fcmp s3, s1
+; CHECK-GI-CVT-NEXT:    asr x1, x0, #63
+; CHECK-GI-CVT-NEXT:    fcvtzs x2, s4
 ; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    fcvtzs x10, s1
-; CHECK-GI-CVT-NEXT:    fcvt s1, h7
-; CHECK-GI-CVT-NEXT:    fcvtzs x11, s2
-; CHECK-GI-CVT-NEXT:    fcvtzs x12, s3
-; CHECK-GI-CVT-NEXT:    mov v2.d[0], x9
-; CHECK-GI-CVT-NEXT:    fcvtzs x9, s4
-; CHECK-GI-CVT-NEXT:    mov v3.d[0], x10
-; CHECK-GI-CVT-NEXT:    fcvtzs x10, s5
-; CHECK-GI-CVT-NEXT:    mov v4.d[0], x11
-; CHECK-GI-CVT-NEXT:    fcvtzs x11, s1
+; CHECK-GI-CVT-NEXT:    csel x0, xzr, x0, lt
+; CHECK-GI-CVT-NEXT:    csel x1, x10, x1, lt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s2
+; CHECK-GI-CVT-NEXT:    csinv x0, x0, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x3, x9, x1, gt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s3
+; CHECK-GI-CVT-NEXT:    fcvt s3, h5
+; CHECK-GI-CVT-NEXT:    fcvtzs x6, s0
+; CHECK-GI-CVT-NEXT:    csel x1, xzr, x0, vs
+; CHECK-GI-CVT-NEXT:    csel x0, xzr, x3, vs
+; CHECK-GI-CVT-NEXT:    fcmp s4, s1
+; CHECK-GI-CVT-NEXT:    asr x3, x2, #63
+; CHECK-GI-CVT-NEXT:    fcvtzs x4, s3
+; CHECK-GI-CVT-NEXT:    asr x7, x6, #63
+; CHECK-GI-CVT-NEXT:    csel x2, xzr, x2, lt
+; CHECK-GI-CVT-NEXT:    csel x3, x10, x3, lt
+; CHECK-GI-CVT-NEXT:    fcmp s4, s2
+; CHECK-GI-CVT-NEXT:    csinv x2, x2, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x3, x9, x3, gt
+; CHECK-GI-CVT-NEXT:    fcmp s4, s4
+; CHECK-GI-CVT-NEXT:    asr x5, x4, #63
+; CHECK-GI-CVT-NEXT:    mov v4.d[0], x18
+; CHECK-GI-CVT-NEXT:    csel x2, xzr, x2, vs
+; CHECK-GI-CVT-NEXT:    csel x3, xzr, x3, vs
+; CHECK-GI-CVT-NEXT:    fcmp s3, s1
+; CHECK-GI-CVT-NEXT:    mov v5.d[0], x2
+; CHECK-GI-CVT-NEXT:    mov v4.d[1], x17
+; CHECK-GI-CVT-NEXT:    csel x4, xzr, x4, lt
+; CHECK-GI-CVT-NEXT:    csel x5, x10, x5, lt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s2
+; CHECK-GI-CVT-NEXT:    mov v5.d[1], x3
+; CHECK-GI-CVT-NEXT:    csinv x4, x4, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x5, x9, x5, gt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s3
+; CHECK-GI-CVT-NEXT:    mov v3.d[0], x16
+; CHECK-GI-CVT-NEXT:    csel x4, xzr, x4, vs
+; CHECK-GI-CVT-NEXT:    csel x5, xzr, x5, vs
+; CHECK-GI-CVT-NEXT:    fcmp s0, s1
 ; CHECK-GI-CVT-NEXT:    mov v1.d[0], x12
-; CHECK-GI-CVT-NEXT:    fcvtzs x12, s0
-; CHECK-GI-CVT-NEXT:    mov v0.d[0], x9
-; CHECK-GI-CVT-NEXT:    mov v2.d[1], xzr
-; CHECK-GI-CVT-NEXT:    mov v5.d[0], x10
-; CHECK-GI-CVT-NEXT:    mov v3.d[1], xzr
-; CHECK-GI-CVT-NEXT:    mov v4.d[1], xzr
-; CHECK-GI-CVT-NEXT:    mov v6.d[0], x11
-; CHECK-GI-CVT-NEXT:    mov v7.d[0], x12
-; CHECK-GI-CVT-NEXT:    mov v1.d[1], xzr
-; CHECK-GI-CVT-NEXT:    mov v0.d[1], xzr
-; CHECK-GI-CVT-NEXT:    mov v5.d[1], xzr
-; CHECK-GI-CVT-NEXT:    stp q2, q3, [x8]
-; CHECK-GI-CVT-NEXT:    mov v6.d[1], xzr
-; CHECK-GI-CVT-NEXT:    mov v7.d[1], xzr
-; CHECK-GI-CVT-NEXT:    stp q4, q1, [x8, #32]
+; CHECK-GI-CVT-NEXT:    mov v6.d[0], x4
+; CHECK-GI-CVT-NEXT:    mov v3.d[1], x15
+; CHECK-GI-CVT-NEXT:    csel x10, x10, x7, lt
+; CHECK-GI-CVT-NEXT:    csel x12, xzr, x6, lt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s2
+; CHECK-GI-CVT-NEXT:    mov v2.d[0], x14
+; CHECK-GI-CVT-NEXT:    mov v1.d[1], x11
+; CHECK-GI-CVT-NEXT:    mov v6.d[1], x5
+; CHECK-GI-CVT-NEXT:    csel x9, x9, x10, gt
+; CHECK-GI-CVT-NEXT:    csinv x10, x12, xzr, le
+; CHECK-GI-CVT-NEXT:    fcmp s0, s0
+; CHECK-GI-CVT-NEXT:    mov v0.d[0], x1
+; CHECK-GI-CVT-NEXT:    stp q3, q4, [x8, #32]
+; CHECK-GI-CVT-NEXT:    mov v2.d[1], x13
+; CHECK-GI-CVT-NEXT:    csel x10, xzr, x10, vs
+; CHECK-GI-CVT-NEXT:    csel x9, xzr, x9, vs
+; CHECK-GI-CVT-NEXT:    mov v7.d[0], x10
+; CHECK-GI-CVT-NEXT:    mov v0.d[1], x0
+; CHECK-GI-CVT-NEXT:    stp q1, q2, [x8]
+; CHECK-GI-CVT-NEXT:    mov v7.d[1], x9
 ; CHECK-GI-CVT-NEXT:    stp q0, q5, [x8, #64]
 ; CHECK-GI-CVT-NEXT:    stp q6, q7, [x8, #96]
 ; CHECK-GI-CVT-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: test_signed_v8f16_v8i128:
 ; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
+; CHECK-GI-FP16-NEXT:    mvni v1.4h, #4, lsl #8
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h0
+; CHECK-GI-FP16-NEXT:    mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-FP16-NEXT:    mvni v2.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    asr x11, x9, #63
+; CHECK-GI-FP16-NEXT:    fcvtzs x13, h3
+; CHECK-GI-FP16-NEXT:    fcvtzs x15, h4
+; CHECK-GI-FP16-NEXT:    csel x12, xzr, x9, lt
+; CHECK-GI-FP16-NEXT:    csel x11, x10, x11, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h2
+; CHECK-GI-FP16-NEXT:    mov x9, #9223372036854775807 // =0x7fffffffffffffff
+; CHECK-GI-FP16-NEXT:    csinv x12, x12, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x11, x9, x11, gt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h0
+; CHECK-GI-FP16-NEXT:    asr x14, x13, #63
+; CHECK-GI-FP16-NEXT:    csel x12, xzr, x12, vs
+; CHECK-GI-FP16-NEXT:    csel x11, xzr, x11, vs
+; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    csel x13, xzr, x13, lt
+; CHECK-GI-FP16-NEXT:    csel x14, x10, x14, lt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h2
+; CHECK-GI-FP16-NEXT:    csinv x13, x13, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x16, x9, x14, gt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h3
 ; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-FP16-NEXT:    csel x14, xzr, x13, vs
+; CHECK-GI-FP16-NEXT:    csel x13, xzr, x16, vs
+; CHECK-GI-FP16-NEXT:    fcmp h4, h1
+; CHECK-GI-FP16-NEXT:    asr x16, x15, #63
+; CHECK-GI-FP16-NEXT:    fcvtzs x17, h3
+; CHECK-GI-FP16-NEXT:    csel x15, xzr, x15, lt
+; CHECK-GI-FP16-NEXT:    csel x16, x10, x16, lt
+; CHECK-GI-FP16-NEXT:    fcmp h4, h2
+; CHECK-GI-FP16-NEXT:    csinv x15, x15, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x18, x9, x16, gt
+; CHECK-GI-FP16-NEXT:    fcmp h4, h4
 ; CHECK-GI-FP16-NEXT:    mov h4, v0.h[4]
-; CHECK-GI-FP16-NEXT:    fcvtzs x9, h0
-; CHECK-GI-FP16-NEXT:    mov h5, v0.h[5]
-; CHECK-GI-FP16-NEXT:    fcvtzs x10, h1
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[6]
-; CHECK-GI-FP16-NEXT:    fcvtzs x11, h2
+; CHECK-GI-FP16-NEXT:    csel x16, xzr, x15, vs
+; CHECK-GI-FP16-NEXT:    csel x15, xzr, x18, vs
+; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    asr x18, x17, #63
+; CHECK-GI-FP16-NEXT:    fcvtzs x0, h4
+; CHECK-GI-FP16-NEXT:    csel x17, xzr, x17, lt
+; CHECK-GI-FP16-NEXT:    csel x18, x10, x18, lt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h2
+; CHECK-GI-FP16-NEXT:    csinv x17, x17, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x1, x9, x18, gt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h3
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[5]
+; CHECK-GI-FP16-NEXT:    csel x18, xzr, x17, vs
+; CHECK-GI-FP16-NEXT:    csel x17, xzr, x1, vs
+; CHECK-GI-FP16-NEXT:    fcmp h4, h1
+; CHECK-GI-FP16-NEXT:    asr x1, x0, #63
+; CHECK-GI-FP16-NEXT:    fcvtzs x2, h3
+; CHECK-GI-FP16-NEXT:    csel x0, xzr, x0, lt
+; CHECK-GI-FP16-NEXT:    csel x1, x10, x1, lt
+; CHECK-GI-FP16-NEXT:    fcmp h4, h2
+; CHECK-GI-FP16-NEXT:    csinv x0, x0, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x1, x9, x1, gt
+; CHECK-GI-FP16-NEXT:    fcmp h4, h4
+; CHECK-GI-FP16-NEXT:    mov h4, v0.h[6]
 ; CHECK-GI-FP16-NEXT:    mov h0, v0.h[7]
-; CHECK-GI-FP16-NEXT:    fcvtzs x12, h3
-; CHECK-GI-FP16-NEXT:    mov v2.d[0], x9
-; CHECK-GI-FP16-NEXT:    fcvtzs x9, h4
-; CHECK-GI-FP16-NEXT:    mov v3.d[0], x10
-; CHECK-GI-FP16-NEXT:    fcvtzs x10, h5
-; CHECK-GI-FP16-NEXT:    mov v4.d[0], x11
-; CHECK-GI-FP16-NEXT:    fcvtzs x11, h1
+; CHECK-GI-FP16-NEXT:    csel x3, xzr, x0, vs
+; CHECK-GI-FP16-NEXT:    csel x0, xzr, x1, vs
+; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    asr x1, x2, #63
+; CHECK-GI-FP16-NEXT:    fcvtzs x4, h4
+; CHECK-GI-FP16-NEXT:    fcvtzs x6, h0
+; CHECK-GI-FP16-NEXT:    csel x2, xzr, x2, lt
+; CHECK-GI-FP16-NEXT:    csel x1, x10, x1, lt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h2
+; CHECK-GI-FP16-NEXT:    csinv x2, x2, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x1, x9, x1, gt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h3
+; CHECK-GI-FP16-NEXT:    asr x5, x4, #63
+; CHECK-GI-FP16-NEXT:    asr x7, x6, #63
+; CHECK-GI-FP16-NEXT:    mov v3.d[0], x16
+; CHECK-GI-FP16-NEXT:    csel x2, xzr, x2, vs
+; CHECK-GI-FP16-NEXT:    csel x1, xzr, x1, vs
+; CHECK-GI-FP16-NEXT:    fcmp h4, h1
+; CHECK-GI-FP16-NEXT:    mov v5.d[0], x2
+; CHECK-GI-FP16-NEXT:    mov v3.d[1], x15
+; CHECK-GI-FP16-NEXT:    csel x4, xzr, x4, lt
+; CHECK-GI-FP16-NEXT:    csel x5, x10, x5, lt
+; CHECK-GI-FP16-NEXT:    fcmp h4, h2
+; CHECK-GI-FP16-NEXT:    mov v5.d[1], x1
+; CHECK-GI-FP16-NEXT:    csinv x4, x4, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x5, x9, x5, gt
+; CHECK-GI-FP16-NEXT:    fcmp h4, h4
+; CHECK-GI-FP16-NEXT:    mov v4.d[0], x18
+; CHECK-GI-FP16-NEXT:    csel x4, xzr, x4, vs
+; CHECK-GI-FP16-NEXT:    csel x5, xzr, x5, vs
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
 ; CHECK-GI-FP16-NEXT:    mov v1.d[0], x12
-; CHECK-GI-FP16-NEXT:    fcvtzs x12, h0
-; CHECK-GI-FP16-NEXT:    mov v0.d[0], x9
-; CHECK-GI-FP16-NEXT:    mov v2.d[1], xzr
-; CHECK-GI-FP16-NEXT:    mov v5.d[0], x10
-; CHECK-GI-FP16-NEXT:    mov v3.d[1], xzr
-; CHECK-GI-FP16-NEXT:    mov v4.d[1], xzr
-; CHECK-GI-FP16-NEXT:    mov v6.d[0], x11
-; CHECK-GI-FP16-NEXT:    mov v7.d[0], x12
-; CHECK-GI-FP16-NEXT:    mov v1.d[1], xzr
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], xzr
-; CHECK-GI-FP16-NEXT:    mov v5.d[1], xzr
-; CHECK-GI-FP16-NEXT:    stp q2, q3, [x8]
-; CHECK-GI-FP16-NEXT:    mov v6.d[1], xzr
-; CHECK-GI-FP16-NEXT:    mov v7.d[1], xzr
-; CHECK-GI-FP16-NEXT:    stp q4, q1, [x8, #32]
+; CHECK-GI-FP16-NEXT:    mov v6.d[0], x4
+; CHECK-GI-FP16-NEXT:    mov v4.d[1], x17
+; CHECK-GI-FP16-NEXT:    csel x10, x10, x7, lt
+; CHECK-GI-FP16-NEXT:    csel x12, xzr, x6, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h2
+; CHECK-GI-FP16-NEXT:    mov v2.d[0], x14
+; CHECK-GI-FP16-NEXT:    mov v1.d[1], x11
+; CHECK-GI-FP16-NEXT:    mov v6.d[1], x5
+; CHECK-GI-FP16-NEXT:    csel x9, x9, x10, gt
+; CHECK-GI-FP16-NEXT:    csinv x10, x12, xzr, le
+; CHECK-GI-FP16-NEXT:    fcmp h0, h0
+; CHECK-GI-FP16-NEXT:    mov v0.d[0], x3
+; CHECK-GI-FP16-NEXT:    stp q3, q4, [x8, #32]
+; CHECK-GI-FP16-NEXT:    mov v2.d[1], x13
+; CHECK-GI-FP16-NEXT:    csel x10, xzr, x10, vs
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    mov v7.d[0], x10
+; CHECK-GI-FP16-NEXT:    mov v0.d[1], x0
+; CHECK-GI-FP16-NEXT:    stp q1, q2, [x8]
+; CHECK-GI-FP16-NEXT:    mov v7.d[1], x9
 ; CHECK-GI-FP16-NEXT:    stp q0, q5, [x8, #64]
 ; CHECK-GI-FP16-NEXT:    stp q6, q7, [x8, #96]
 ; CHECK-GI-FP16-NEXT:    ret
diff --git a/llvm/test/CodeGen/AArch64/fptoui-sat-scalar.ll b/llvm/test/CodeGen/AArch64/fptoui-sat-scalar.ll
index 7a5fe0f4222bc0..a503f54a33f6e1 100644
--- a/llvm/test/CodeGen/AArch64/fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/AArch64/fptoui-sat-scalar.ll
@@ -705,14 +705,28 @@ define i100 @test_unsigned_i100_f16(half %f) nounwind {
 ; CHECK-GI-CVT-LABEL: test_unsigned_i100_f16:
 ; CHECK-GI-CVT:       // %bb.0:
 ; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    mov x1, xzr
-; CHECK-GI-CVT-NEXT:    fcvtzu x0, s0
+; CHECK-GI-CVT-NEXT:    mov w9, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    movk w9, #18303, lsl #16
+; CHECK-GI-CVT-NEXT:    fmov s1, w9
+; CHECK-GI-CVT-NEXT:    mov x9, #68719476735 // =0xfffffffff
+; CHECK-GI-CVT-NEXT:    fcvtzu x8, s0
+; CHECK-GI-CVT-NEXT:    fcmp s0, #0.0
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s1
+; CHECK-GI-CVT-NEXT:    csinv x0, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x1, x9, xzr, gt
 ; CHECK-GI-CVT-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: test_unsigned_i100_f16:
 ; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzu x0, h0
-; CHECK-GI-FP16-NEXT:    mov x1, xzr
+; CHECK-GI-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    mov x9, #68719476735 // =0xfffffffff
+; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    csinv x0, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x1, x9, xzr, gt
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call i100 @llvm.fptoui.sat.i100.f16(half %f)
     ret i100 %x
@@ -741,14 +755,26 @@ define i128 @test_unsigned_i128_f16(half %f) nounwind {
 ; CHECK-GI-CVT-LABEL: test_unsigned_i128_f16:
 ; CHECK-GI-CVT:       // %bb.0:
 ; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    mov x1, xzr
-; CHECK-GI-CVT-NEXT:    fcvtzu x0, s0
+; CHECK-GI-CVT-NEXT:    mov w8, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    movk w8, #18303, lsl #16
+; CHECK-GI-CVT-NEXT:    fmov s1, w8
+; CHECK-GI-CVT-NEXT:    fcvtzu x9, s0
+; CHECK-GI-CVT-NEXT:    fcmp s0, #0.0
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x9, lt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s1
+; CHECK-GI-CVT-NEXT:    csinv x0, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csetm x1, gt
 ; CHECK-GI-CVT-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: test_unsigned_i128_f16:
 ; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzu x0, h0
-; CHECK-GI-FP16-NEXT:    mov x1, xzr
+; CHECK-GI-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    csinv x0, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csetm x1, gt
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call i128 @llvm.fptoui.sat.i128.f16(half %f)
     ret i128 %x
diff --git a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
index 083a84292029d7..bc743cb31f1947 100644
--- a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
@@ -2587,37 +2587,75 @@ define <4 x i100> @test_unsigned_v4f16_v4i100(<4 x half> %f) {
 ; CHECK-GI-CVT-LABEL: test_unsigned_v4f16_v4i100:
 ; CHECK-GI-CVT:       // %bb.0:
 ; CHECK-GI-CVT-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-CVT-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-CVT-NEXT:    mov x1, xzr
-; CHECK-GI-CVT-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    mov x3, xzr
-; CHECK-GI-CVT-NEXT:    mov x5, xzr
-; CHECK-GI-CVT-NEXT:    mov x7, xzr
-; CHECK-GI-CVT-NEXT:    fcvt s1, h1
+; CHECK-GI-CVT-NEXT:    fcvt s1, h0
+; CHECK-GI-CVT-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-CVT-NEXT:    mov w8, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    movk w8, #18303, lsl #16
+; CHECK-GI-CVT-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-CVT-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-CVT-NEXT:    fmov s3, w8
+; CHECK-GI-CVT-NEXT:    fcvtzu x9, s1
+; CHECK-GI-CVT-NEXT:    fcmp s1, #0.0
 ; CHECK-GI-CVT-NEXT:    fcvt s2, h2
-; CHECK-GI-CVT-NEXT:    fcvt s3, h3
-; CHECK-GI-CVT-NEXT:    fcvtzu x0, s0
-; CHECK-GI-CVT-NEXT:    fcvtzu x2, s1
-; CHECK-GI-CVT-NEXT:    fcvtzu x4, s2
-; CHECK-GI-CVT-NEXT:    fcvtzu x6, s3
+; CHECK-GI-CVT-NEXT:    fcvt s0, h0
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x9, lt
+; CHECK-GI-CVT-NEXT:    fcmp s1, s3
+; CHECK-GI-CVT-NEXT:    mov x9, #68719476735 // =0xfffffffff
+; CHECK-GI-CVT-NEXT:    fcvtzu x10, s2
+; CHECK-GI-CVT-NEXT:    fcvt s1, h4
+; CHECK-GI-CVT-NEXT:    csinv x0, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x1, x9, xzr, gt
+; CHECK-GI-CVT-NEXT:    fcmp s2, #0.0
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x10, lt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s3
+; CHECK-GI-CVT-NEXT:    fcvtzu x10, s1
+; CHECK-GI-CVT-NEXT:    csinv x2, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x3, x9, xzr, gt
+; CHECK-GI-CVT-NEXT:    fcmp s1, #0.0
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x10, lt
+; CHECK-GI-CVT-NEXT:    fcmp s1, s3
+; CHECK-GI-CVT-NEXT:    fcvtzu x10, s0
+; CHECK-GI-CVT-NEXT:    csinv x4, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x5, x9, xzr, gt
+; CHECK-GI-CVT-NEXT:    fcmp s0, #0.0
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x10, lt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s3
+; CHECK-GI-CVT-NEXT:    csinv x6, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csel x7, x9, xzr, gt
 ; CHECK-GI-CVT-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: test_unsigned_v4f16_v4i100:
 ; CHECK-GI-FP16:       // %bb.0:
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov x1, xzr
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT:    fcvtzu x0, h0
-; CHECK-GI-FP16-NEXT:    mov x3, xzr
-; CHECK-GI-FP16-NEXT:    mov x5, xzr
-; CHECK-GI-FP16-NEXT:    mov x7, xzr
-; CHECK-GI-FP16-NEXT:    fcvtzu x2, h1
-; CHECK-GI-FP16-NEXT:    fcvtzu x4, h2
-; CHECK-GI-FP16-NEXT:    fcvtzu x6, h3
+; CHECK-GI-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    mov x9, #68719476735 // =0xfffffffff
+; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
+; CHECK-GI-FP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[2]
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h2
+; CHECK-GI-FP16-NEXT:    csinv x0, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x1, x9, xzr, gt
+; CHECK-GI-FP16-NEXT:    fcmp h2, #0.0
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x10, lt
+; CHECK-GI-FP16-NEXT:    fcmp h2, h1
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h3
+; CHECK-GI-FP16-NEXT:    csinv x2, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x3, x9, xzr, gt
+; CHECK-GI-FP16-NEXT:    fcmp h3, #0.0
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x10, lt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h0
+; CHECK-GI-FP16-NEXT:    csinv x4, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x5, x9, xzr, gt
+; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x10, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    csinv x6, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x7, x9, xzr, gt
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call <4 x i100> @llvm.fptoui.sat.v4f16.v4i100(<4 x half> %f)
     ret <4 x i100> %x
@@ -2705,37 +2743,73 @@ define <4 x i128> @test_unsigned_v4f16_v4i128(<4 x half> %f) {
 ; CHECK-GI-CVT-LABEL: test_unsigned_v4f16_v4i128:
 ; CHECK-GI-CVT:       // %bb.0:
 ; CHECK-GI-CVT-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-CVT-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-CVT-NEXT:    mov x1, xzr
-; CHECK-GI-CVT-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    mov x3, xzr
-; CHECK-GI-CVT-NEXT:    mov x5, xzr
-; CHECK-GI-CVT-NEXT:    mov x7, xzr
-; CHECK-GI-CVT-NEXT:    fcvt s1, h1
+; CHECK-GI-CVT-NEXT:    fcvt s1, h0
+; CHECK-GI-CVT-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-CVT-NEXT:    mov w8, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    movk w8, #18303, lsl #16
+; CHECK-GI-CVT-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-CVT-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-CVT-NEXT:    fmov s3, w8
+; CHECK-GI-CVT-NEXT:    fcvtzu x9, s1
+; CHECK-GI-CVT-NEXT:    fcmp s1, #0.0
 ; CHECK-GI-CVT-NEXT:    fcvt s2, h2
-; CHECK-GI-CVT-NEXT:    fcvt s3, h3
-; CHECK-GI-CVT-NEXT:    fcvtzu x0, s0
-; CHECK-GI-CVT-NEXT:    fcvtzu x2, s1
-; CHECK-GI-CVT-NEXT:    fcvtzu x4, s2
-; CHECK-GI-CVT-NEXT:    fcvtzu x6, s3
+; CHECK-GI-CVT-NEXT:    fcvt s0, h0
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x9, lt
+; CHECK-GI-CVT-NEXT:    fcmp s1, s3
+; CHECK-GI-CVT-NEXT:    fcvtzu x9, s2
+; CHECK-GI-CVT-NEXT:    fcvt s1, h4
+; CHECK-GI-CVT-NEXT:    csinv x0, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csetm x1, gt
+; CHECK-GI-CVT-NEXT:    fcmp s2, #0.0
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x9, lt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s3
+; CHECK-GI-CVT-NEXT:    fcvtzu x9, s1
+; CHECK-GI-CVT-NEXT:    csinv x2, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csetm x3, gt
+; CHECK-GI-CVT-NEXT:    fcmp s1, #0.0
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x9, lt
+; CHECK-GI-CVT-NEXT:    fcmp s1, s3
+; CHECK-GI-CVT-NEXT:    fcvtzu x9, s0
+; CHECK-GI-CVT-NEXT:    csinv x4, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csetm x5, gt
+; CHECK-GI-CVT-NEXT:    fcmp s0, #0.0
+; CHECK-GI-CVT-NEXT:    csel x8, xzr, x9, lt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s3
+; CHECK-GI-CVT-NEXT:    csinv x6, x8, xzr, le
+; CHECK-GI-CVT-NEXT:    csetm x7, gt
 ; CHECK-GI-CVT-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: test_unsigned_v4f16_v4i128:
 ; CHECK-GI-FP16:       // %bb.0:
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov x1, xzr
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT:    fcvtzu x0, h0
-; CHECK-GI-FP16-NEXT:    mov x3, xzr
-; CHECK-GI-FP16-NEXT:    mov x5, xzr
-; CHECK-GI-FP16-NEXT:    mov x7, xzr
-; CHECK-GI-FP16-NEXT:    fcvtzu x2, h1
-; CHECK-GI-FP16-NEXT:    fcvtzu x4, h2
-; CHECK-GI-FP16-NEXT:    fcvtzu x6, h3
+; CHECK-GI-FP16-NEXT:    fcvtzu x8, h0
+; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
+; CHECK-GI-FP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[2]
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    mov h0, v0.h[3]
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h2
+; CHECK-GI-FP16-NEXT:    csinv x0, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csetm x1, gt
+; CHECK-GI-FP16-NEXT:    fcmp h2, #0.0
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x9, lt
+; CHECK-GI-FP16-NEXT:    fcmp h2, h1
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h3
+; CHECK-GI-FP16-NEXT:    csinv x2, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csetm x3, gt
+; CHECK-GI-FP16-NEXT:    fcmp h3, #0.0
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x9, lt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h0
+; CHECK-GI-FP16-NEXT:    csinv x4, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csetm x5, gt
+; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x9, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    csinv x6, x8, xzr, le
+; CHECK-GI-FP16-NEXT:    csetm x7, gt
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call <4 x i128> @llvm.fptoui.sat.v4f16.v4i128(<4 x half> %f)
     ret <4 x i128> %x
@@ -3409,87 +3483,171 @@ define <8 x i128> @test_unsigned_v8f16_v8i128(<8 x half> %f) {
 ;
 ; CHECK-GI-CVT-LABEL: test_unsigned_v8f16_v8i128:
 ; CHECK-GI-CVT:       // %bb.0:
+; CHECK-GI-CVT-NEXT:    fcvt s2, h0
 ; CHECK-GI-CVT-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-CVT-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-CVT-NEXT:    mov h3, v0.h[3]
+; CHECK-GI-CVT-NEXT:    mov w9, #57344 // =0xe000
+; CHECK-GI-CVT-NEXT:    movk w9, #18303, lsl #16
+; CHECK-GI-CVT-NEXT:    mov h4, v0.h[2]
+; CHECK-GI-CVT-NEXT:    fcvtzu x10, s2
+; CHECK-GI-CVT-NEXT:    fcmp s2, #0.0
+; CHECK-GI-CVT-NEXT:    fcvt s3, h1
+; CHECK-GI-CVT-NEXT:    fmov s1, w9
+; CHECK-GI-CVT-NEXT:    csel x9, xzr, x10, lt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s1
+; CHECK-GI-CVT-NEXT:    fcvtzu x11, s3
+; CHECK-GI-CVT-NEXT:    fcvt s2, h4
+; CHECK-GI-CVT-NEXT:    mov h4, v0.h[3]
+; CHECK-GI-CVT-NEXT:    csinv x10, x9, xzr, le
+; CHECK-GI-CVT-NEXT:    csetm x9, gt
+; CHECK-GI-CVT-NEXT:    fcmp s3, #0.0
+; CHECK-GI-CVT-NEXT:    fcvtzu x13, s2
+; CHECK-GI-CVT-NEXT:    csel x11, xzr, x11, lt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s1
+; CHECK-GI-CVT-NEXT:    fcvt s3, h4
 ; CHECK-GI-CVT-NEXT:    mov h4, v0.h[4]
-; CHECK-GI-CVT-NEXT:    fcvt s5, h0
-; CHECK-GI-CVT-NEXT:    mov h6, v0.h[5]
-; CHECK-GI-CVT-NEXT:    mov h7, v0.h[6]
+; CHECK-GI-CVT-NEXT:    csinv x12, x11, xzr, le
+; CHECK-GI-CVT-NEXT:    csetm x11, gt
+; CHECK-GI-CVT-NEXT:    fcmp s2, #0.0
+; CHECK-GI-CVT-NEXT:    fcvtzu x15, s3
+; CHECK-GI-CVT-NEXT:    csel x13, xzr, x13, lt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s1
+; CHECK-GI-CVT-NEXT:    fcvt s2, h4
+; CHECK-GI-CVT-NEXT:    mov h4, v0.h[5]
+; CHECK-GI-CVT-NEXT:    csinv x14, x13, xzr, le
+; CHECK-GI-CVT-NEXT:    csetm x13, gt
+; CHECK-GI-CVT-NEXT:    fcmp s3, #0.0
+; CHECK-GI-CVT-NEXT:    fcvtzu x16, s2
+; CHECK-GI-CVT-NEXT:    csel x15, xzr, x15, lt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s1
+; CHECK-GI-CVT-NEXT:    fcvt s3, h4
+; CHECK-GI-CVT-NEXT:    mov h4, v0.h[6]
 ; CHECK-GI-CVT-NEXT:    mov h0, v0.h[7]
-; CHECK-GI-CVT-NEXT:    fcvt s1, h1
-; CHECK-GI-CVT-NEXT:    fcvt s2, h2
-; CHECK-GI-CVT-NEXT:    fcvt s3, h3
-; CHECK-GI-CVT-NEXT:    fcvtzu x9, s5
-; CHECK-GI-CVT-NEXT:    fcvt s4, h4
-; CHECK-GI-CVT-NEXT:    fcvt s5, h6
+; CHECK-GI-CVT-NEXT:    csinv x15, x15, xzr, le
+; CHECK-GI-CVT-NEXT:    csetm x17, gt
+; CHECK-GI-CVT-NEXT:    fcmp s2, #0.0
+; CHECK-GI-CVT-NEXT:    fcvtzu x18, s3
+; CHECK-GI-CVT-NEXT:    mov v5.d[0], x15
 ; CHECK-GI-CVT-NEXT:    fcvt s0, h0
-; CHECK-GI-CVT-NEXT:    fcvtzu x10, s1
-; CHECK-GI-CVT-NEXT:    fcvt s1, h7
-; CHECK-GI-CVT-NEXT:    fcvtzu x11, s2
-; CHECK-GI-CVT-NEXT:    fcvtzu x12, s3
-; CHECK-GI-CVT-NEXT:    mov v2.d[0], x9
-; CHECK-GI-CVT-NEXT:    fcvtzu x9, s4
-; CHECK-GI-CVT-NEXT:    mov v3.d[0], x10
-; CHECK-GI-CVT-NEXT:    fcvtzu x10, s5
-; CHECK-GI-CVT-NEXT:    mov v4.d[0], x11
-; CHECK-GI-CVT-NEXT:    fcvtzu x11, s1
-; CHECK-GI-CVT-NEXT:    mov v1.d[0], x12
-; CHECK-GI-CVT-NEXT:    fcvtzu x12, s0
-; CHECK-GI-CVT-NEXT:    mov v0.d[0], x9
-; CHECK-GI-CVT-NEXT:    mov v2.d[1], xzr
-; CHECK-GI-CVT-NEXT:    mov v5.d[0], x10
-; CHECK-GI-CVT-NEXT:    mov v3.d[1], xzr
-; CHECK-GI-CVT-NEXT:    mov v4.d[1], xzr
-; CHECK-GI-CVT-NEXT:    mov v6.d[0], x11
-; CHECK-GI-CVT-NEXT:    mov v7.d[0], x12
-; CHECK-GI-CVT-NEXT:    mov v1.d[1], xzr
-; CHECK-GI-CVT-NEXT:    mov v0.d[1], xzr
-; CHECK-GI-CVT-NEXT:    mov v5.d[1], xzr
+; CHECK-GI-CVT-NEXT:    csel x16, xzr, x16, lt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s1
+; CHECK-GI-CVT-NEXT:    fcvt s2, h4
+; CHECK-GI-CVT-NEXT:    mov v4.d[0], x14
+; CHECK-GI-CVT-NEXT:    mov v5.d[1], x17
+; CHECK-GI-CVT-NEXT:    csinv x16, x16, xzr, le
+; CHECK-GI-CVT-NEXT:    csetm x0, gt
+; CHECK-GI-CVT-NEXT:    fcmp s3, #0.0
+; CHECK-GI-CVT-NEXT:    fcvtzu x1, s2
+; CHECK-GI-CVT-NEXT:    mov v4.d[1], x13
+; CHECK-GI-CVT-NEXT:    csel x18, xzr, x18, lt
+; CHECK-GI-CVT-NEXT:    fcmp s3, s1
+; CHECK-GI-CVT-NEXT:    mov v3.d[0], x12
+; CHECK-GI-CVT-NEXT:    csinv x18, x18, xzr, le
+; CHECK-GI-CVT-NEXT:    csetm x2, gt
+; CHECK-GI-CVT-NEXT:    fcmp s2, #0.0
+; CHECK-GI-CVT-NEXT:    mov v3.d[1], x11
+; CHECK-GI-CVT-NEXT:    stp q4, q5, [x8, #32]
+; CHECK-GI-CVT-NEXT:    csel x1, xzr, x1, lt
+; CHECK-GI-CVT-NEXT:    fcmp s2, s1
+; CHECK-GI-CVT-NEXT:    mov v2.d[0], x10
+; CHECK-GI-CVT-NEXT:    fcvtzu x10, s0
+; CHECK-GI-CVT-NEXT:    csinv x1, x1, xzr, le
+; CHECK-GI-CVT-NEXT:    csetm x3, gt
+; CHECK-GI-CVT-NEXT:    fcmp s0, #0.0
+; CHECK-GI-CVT-NEXT:    mov v2.d[1], x9
+; CHECK-GI-CVT-NEXT:    mov v6.d[0], x1
+; CHECK-GI-CVT-NEXT:    csel x10, xzr, x10, lt
+; CHECK-GI-CVT-NEXT:    fcmp s0, s1
+; CHECK-GI-CVT-NEXT:    mov v0.d[0], x16
+; CHECK-GI-CVT-NEXT:    mov v1.d[0], x18
+; CHECK-GI-CVT-NEXT:    mov v6.d[1], x3
 ; CHECK-GI-CVT-NEXT:    stp q2, q3, [x8]
-; CHECK-GI-CVT-NEXT:    mov v6.d[1], xzr
-; CHECK-GI-CVT-NEXT:    mov v7.d[1], xzr
-; CHECK-GI-CVT-NEXT:    stp q4, q1, [x8, #32]
-; CHECK-GI-CVT-NEXT:    stp q0, q5, [x8, #64]
+; CHECK-GI-CVT-NEXT:    csinv x9, x10, xzr, le
+; CHECK-GI-CVT-NEXT:    mov v7.d[0], x9
+; CHECK-GI-CVT-NEXT:    csetm x9, gt
+; CHECK-GI-CVT-NEXT:    mov v0.d[1], x0
+; CHECK-GI-CVT-NEXT:    mov v1.d[1], x2
+; CHECK-GI-CVT-NEXT:    mov v7.d[1], x9
+; CHECK-GI-CVT-NEXT:    stp q0, q1, [x8, #64]
 ; CHECK-GI-CVT-NEXT:    stp q6, q7, [x8, #96]
 ; CHECK-GI-CVT-NEXT:    ret
 ;
 ; CHECK-GI-FP16-LABEL: test_unsigned_v8f16_v8i128:
 ; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[2]
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[3]
-; CHECK-GI-FP16-NEXT:    mov h4, v0.h[4]
 ; CHECK-GI-FP16-NEXT:    fcvtzu x9, h0
-; CHECK-GI-FP16-NEXT:    mov h5, v0.h[5]
-; CHECK-GI-FP16-NEXT:    fcvtzu x10, h1
-; CHECK-GI-FP16-NEXT:    mov h1, v0.h[6]
-; CHECK-GI-FP16-NEXT:    fcvtzu x11, h2
+; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
+; CHECK-GI-FP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[2]
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x9, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h2
+; CHECK-GI-FP16-NEXT:    fcvtzu x13, h3
+; CHECK-GI-FP16-NEXT:    csinv x11, x9, xzr, le
+; CHECK-GI-FP16-NEXT:    csetm x9, gt
+; CHECK-GI-FP16-NEXT:    fcmp h2, #0.0
+; CHECK-GI-FP16-NEXT:    csel x10, xzr, x10, lt
+; CHECK-GI-FP16-NEXT:    fcmp h2, h1
+; CHECK-GI-FP16-NEXT:    mov h2, v0.h[3]
+; CHECK-GI-FP16-NEXT:    csinv x12, x10, xzr, le
+; CHECK-GI-FP16-NEXT:    csetm x10, gt
+; CHECK-GI-FP16-NEXT:    fcmp h3, #0.0
+; CHECK-GI-FP16-NEXT:    fcvtzu x14, h2
+; CHECK-GI-FP16-NEXT:    csel x13, xzr, x13, lt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[4]
+; CHECK-GI-FP16-NEXT:    csinv x13, x13, xzr, le
+; CHECK-GI-FP16-NEXT:    csetm x15, gt
+; CHECK-GI-FP16-NEXT:    fcmp h2, #0.0
+; CHECK-GI-FP16-NEXT:    fcvtzu x16, h3
+; CHECK-GI-FP16-NEXT:    mov v4.d[0], x13
+; CHECK-GI-FP16-NEXT:    csel x14, xzr, x14, lt
+; CHECK-GI-FP16-NEXT:    fcmp h2, h1
+; CHECK-GI-FP16-NEXT:    mov h2, v0.h[5]
+; CHECK-GI-FP16-NEXT:    mov v4.d[1], x15
+; CHECK-GI-FP16-NEXT:    csinv x14, x14, xzr, le
+; CHECK-GI-FP16-NEXT:    csetm x17, gt
+; CHECK-GI-FP16-NEXT:    fcmp h3, #0.0
+; CHECK-GI-FP16-NEXT:    fcvtzu x18, h2
+; CHECK-GI-FP16-NEXT:    mov v5.d[0], x14
+; CHECK-GI-FP16-NEXT:    csel x16, xzr, x16, lt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    mov h3, v0.h[6]
 ; CHECK-GI-FP16-NEXT:    mov h0, v0.h[7]
-; CHECK-GI-FP16-NEXT:    fcvtzu x12, h3
-; CHECK-GI-FP16-NEXT:    mov v2.d[0], x9
-; CHECK-GI-FP16-NEXT:    fcvtzu x9, h4
-; CHECK-GI-FP16-NEXT:    mov v3.d[0], x10
-; CHECK-GI-FP16-NEXT:    fcvtzu x10, h5
-; CHECK-GI-FP16-NEXT:    mov v4.d[0], x11
-; CHECK-GI-FP16-NEXT:    fcvtzu x11, h1
-; CHECK-GI-FP16-NEXT:    mov v1.d[0], x12
-; CHECK-GI-FP16-NEXT:    fcvtzu x12, h0
-; CHECK-GI-FP16-NEXT:    mov v0.d[0], x9
-; CHECK-GI-FP16-NEXT:    mov v2.d[1], xzr
-; CHECK-GI-FP16-NEXT:    mov v5.d[0], x10
-; CHECK-GI-FP16-NEXT:    mov v3.d[1], xzr
-; CHECK-GI-FP16-NEXT:    mov v4.d[1], xzr
-; CHECK-GI-FP16-NEXT:    mov v6.d[0], x11
-; CHECK-GI-FP16-NEXT:    mov v7.d[0], x12
-; CHECK-GI-FP16-NEXT:    mov v1.d[1], xzr
-; CHECK-GI-FP16-NEXT:    mov v0.d[1], xzr
-; CHECK-GI-FP16-NEXT:    mov v5.d[1], xzr
+; CHECK-GI-FP16-NEXT:    mov v5.d[1], x17
+; CHECK-GI-FP16-NEXT:    csinv x16, x16, xzr, le
+; CHECK-GI-FP16-NEXT:    csetm x0, gt
+; CHECK-GI-FP16-NEXT:    fcmp h2, #0.0
+; CHECK-GI-FP16-NEXT:    fcvtzu x1, h3
+; CHECK-GI-FP16-NEXT:    csel x18, xzr, x18, lt
+; CHECK-GI-FP16-NEXT:    fcmp h2, h1
+; CHECK-GI-FP16-NEXT:    mov v2.d[0], x11
+; CHECK-GI-FP16-NEXT:    fcvtzu x11, h0
+; CHECK-GI-FP16-NEXT:    stp q4, q5, [x8, #32]
+; CHECK-GI-FP16-NEXT:    csinv x18, x18, xzr, le
+; CHECK-GI-FP16-NEXT:    csetm x2, gt
+; CHECK-GI-FP16-NEXT:    fcmp h3, #0.0
+; CHECK-GI-FP16-NEXT:    mov v2.d[1], x9
+; CHECK-GI-FP16-NEXT:    csel x1, xzr, x1, lt
+; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    mov v3.d[0], x12
+; CHECK-GI-FP16-NEXT:    csinv x1, x1, xzr, le
+; CHECK-GI-FP16-NEXT:    csetm x3, gt
+; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
+; CHECK-GI-FP16-NEXT:    mov v6.d[0], x1
+; CHECK-GI-FP16-NEXT:    mov v3.d[1], x10
+; CHECK-GI-FP16-NEXT:    csel x11, xzr, x11, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    mov v0.d[0], x16
+; CHECK-GI-FP16-NEXT:    mov v1.d[0], x18
+; CHECK-GI-FP16-NEXT:    mov v6.d[1], x3
 ; CHECK-GI-FP16-NEXT:    stp q2, q3, [x8]
-; CHECK-GI-FP16-NEXT:    mov v6.d[1], xzr
-; CHECK-GI-FP16-NEXT:    mov v7.d[1], xzr
-; CHECK-GI-FP16-NEXT:    stp q4, q1, [x8, #32]
-; CHECK-GI-FP16-NEXT:    stp q0, q5, [x8, #64]
+; CHECK-GI-FP16-NEXT:    csinv x9, x11, xzr, le
+; CHECK-GI-FP16-NEXT:    mov v7.d[0], x9
+; CHECK-GI-FP16-NEXT:    csetm x9, gt
+; CHECK-GI-FP16-NEXT:    mov v0.d[1], x0
+; CHECK-GI-FP16-NEXT:    mov v1.d[1], x2
+; CHECK-GI-FP16-NEXT:    mov v7.d[1], x9
+; CHECK-GI-FP16-NEXT:    stp q0, q1, [x8, #64]
 ; CHECK-GI-FP16-NEXT:    stp q6, q7, [x8, #96]
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call <8 x i128> @llvm.fptoui.sat.v8f16.v8i128(<8 x half> %f)

>From e3192650de2af8ce593ac4efbf91012a8ec2a2b5 Mon Sep 17 00:00:00 2001
From: Folkert de Vries <folkert at folkertdev.nl>
Date: Sun, 8 Nov 2026 16:37:34 +0100
Subject: [PATCH 2/2] update CHECKs

---
 .../test/CodeGen/AArch64/fptosi-sat-scalar.ll |  48 ++++----
 .../test/CodeGen/AArch64/fptosi-sat-vector.ll | 104 +++++++++---------
 .../test/CodeGen/AArch64/fptoui-sat-scalar.ll |  16 +--
 .../test/CodeGen/AArch64/fptoui-sat-vector.ll |  51 +++++----
 4 files changed, 117 insertions(+), 102 deletions(-)

diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll
index d2376885f94de5..921e87f15c2724 100644
--- a/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll
+++ b/llvm/test/CodeGen/AArch64/fptosi-sat-scalar.ll
@@ -900,21 +900,23 @@ define i100 @test_signed_i100_f16(half %f) nounwind {
 ;
 ; CHECK-GI-FP16-LABEL: test_signed_i100_f16:
 ; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #4, lsl #8
-; CHECK-GI-FP16-NEXT:    fcvtzs x8, h0
-; CHECK-GI-FP16-NEXT:    mov x10, #34359738368 // =0x800000000
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI28_1
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h0
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI28_1]
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI28_0
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h1
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
-; CHECK-GI-FP16-NEXT:    asr x9, x8, #63
-; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
-; CHECK-GI-FP16-NEXT:    csel x9, x10, x9, lt
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI28_0]
+; CHECK-GI-FP16-NEXT:    mov x8, #34359738368 // =0x800000000
+; CHECK-GI-FP16-NEXT:    asr x10, x9, #63
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x9, lt
+; CHECK-GI-FP16-NEXT:    csel x8, x8, x10, lt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h1
 ; CHECK-GI-FP16-NEXT:    mov x10, #34359738367 // =0x7ffffffff
-; CHECK-GI-FP16-NEXT:    csinv x8, x8, xzr, le
-; CHECK-GI-FP16-NEXT:    csel x9, x10, x9, gt
+; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, gt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h0
-; CHECK-GI-FP16-NEXT:    csel x0, xzr, x8, vs
-; CHECK-GI-FP16-NEXT:    csel x1, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    csel x0, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call i100 @llvm.fptosi.sat.i100.f16(half %f)
     ret i100 %x
@@ -972,21 +974,23 @@ define i128 @test_signed_i128_f16(half %f) nounwind {
 ;
 ; CHECK-GI-FP16-LABEL: test_signed_i128_f16:
 ; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #4, lsl #8
-; CHECK-GI-FP16-NEXT:    fcvtzs x8, h0
-; CHECK-GI-FP16-NEXT:    mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI29_1
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h0
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI29_1]
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI29_0
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h1
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
-; CHECK-GI-FP16-NEXT:    asr x9, x8, #63
-; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
-; CHECK-GI-FP16-NEXT:    csel x9, x10, x9, lt
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI29_0]
+; CHECK-GI-FP16-NEXT:    mov x8, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-FP16-NEXT:    asr x10, x9, #63
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x9, lt
+; CHECK-GI-FP16-NEXT:    csel x8, x8, x10, lt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h1
 ; CHECK-GI-FP16-NEXT:    mov x10, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-GI-FP16-NEXT:    csinv x8, x8, xzr, le
-; CHECK-GI-FP16-NEXT:    csel x9, x10, x9, gt
+; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, gt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h0
-; CHECK-GI-FP16-NEXT:    csel x0, xzr, x8, vs
-; CHECK-GI-FP16-NEXT:    csel x1, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    csel x0, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    csel x1, xzr, x8, vs
 ; CHECK-GI-FP16-NEXT:    ret
     %x = call i128 @llvm.fptosi.sat.i128.f16(half %f)
     ret i128 %x
diff --git a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
index b756cc31f57a5c..fcceef66470c7b 100644
--- a/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptosi-sat-vector.ll
@@ -3301,41 +3301,43 @@ define <4 x i100> @test_signed_v4f16_v4i100(<4 x half> %f) {
 ; CHECK-GI-FP16-LABEL: test_signed_v4f16_v4i100:
 ; CHECK-GI-FP16:       // %bb.0:
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #4, lsl #8
-; CHECK-GI-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI64_1
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h0
+; CHECK-GI-FP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI64_1]
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI64_0
 ; CHECK-GI-FP16-NEXT:    mov x10, #34359738368 // =0x800000000
-; CHECK-GI-FP16-NEXT:    mvni v2.4h, #132, lsl #8
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-FP16-NEXT:    ldr h3, [x8, :lo12:.LCPI64_0]
 ; CHECK-GI-FP16-NEXT:    mov x11, #34359738367 // =0x7ffffffff
 ; CHECK-GI-FP16-NEXT:    mov h4, v0.h[2]
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h1
-; CHECK-GI-FP16-NEXT:    asr x9, x8, #63
-; CHECK-GI-FP16-NEXT:    fcvtzs x12, h3
-; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
-; CHECK-GI-FP16-NEXT:    csel x9, x10, x9, lt
-; CHECK-GI-FP16-NEXT:    fcmp h0, h2
-; CHECK-GI-FP16-NEXT:    csinv x8, x8, xzr, le
-; CHECK-GI-FP16-NEXT:    csel x9, x11, x9, gt
+; CHECK-GI-FP16-NEXT:    asr x8, x9, #63
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h2
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x9, lt
+; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h3
+; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h0
 ; CHECK-GI-FP16-NEXT:    mov h0, v0.h[3]
-; CHECK-GI-FP16-NEXT:    csel x0, xzr, x8, vs
-; CHECK-GI-FP16-NEXT:    csel x1, xzr, x9, vs
-; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    csel x0, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    csel x1, xzr, x8, vs
+; CHECK-GI-FP16-NEXT:    fcmp h2, h1
 ; CHECK-GI-FP16-NEXT:    asr x8, x12, #63
 ; CHECK-GI-FP16-NEXT:    csel x9, xzr, x12, lt
 ; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
-; CHECK-GI-FP16-NEXT:    fcmp h3, h2
+; CHECK-GI-FP16-NEXT:    fcmp h2, h3
 ; CHECK-GI-FP16-NEXT:    fcvtzs x12, h4
 ; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
 ; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
-; CHECK-GI-FP16-NEXT:    fcmp h3, h3
+; CHECK-GI-FP16-NEXT:    fcmp h2, h2
 ; CHECK-GI-FP16-NEXT:    csel x2, xzr, x9, vs
 ; CHECK-GI-FP16-NEXT:    csel x3, xzr, x8, vs
 ; CHECK-GI-FP16-NEXT:    fcmp h4, h1
 ; CHECK-GI-FP16-NEXT:    asr x8, x12, #63
 ; CHECK-GI-FP16-NEXT:    csel x9, xzr, x12, lt
 ; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
-; CHECK-GI-FP16-NEXT:    fcmp h4, h2
+; CHECK-GI-FP16-NEXT:    fcmp h4, h3
 ; CHECK-GI-FP16-NEXT:    fcvtzs x12, h0
 ; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
 ; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
@@ -3346,7 +3348,7 @@ define <4 x i100> @test_signed_v4f16_v4i100(<4 x half> %f) {
 ; CHECK-GI-FP16-NEXT:    asr x8, x12, #63
 ; CHECK-GI-FP16-NEXT:    csel x9, xzr, x12, lt
 ; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
-; CHECK-GI-FP16-NEXT:    fcmp h0, h2
+; CHECK-GI-FP16-NEXT:    fcmp h0, h3
 ; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
 ; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h0
@@ -3525,41 +3527,43 @@ define <4 x i128> @test_signed_v4f16_v4i128(<4 x half> %f) {
 ; CHECK-GI-FP16-LABEL: test_signed_v4f16_v4i128:
 ; CHECK-GI-FP16:       // %bb.0:
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #4, lsl #8
-; CHECK-GI-FP16-NEXT:    fcvtzs x8, h0
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI65_1
+; CHECK-GI-FP16-NEXT:    fcvtzs x9, h0
+; CHECK-GI-FP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI65_1]
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI65_0
 ; CHECK-GI-FP16-NEXT:    mov x10, #-9223372036854775808 // =0x8000000000000000
-; CHECK-GI-FP16-NEXT:    mvni v2.4h, #132, lsl #8
-; CHECK-GI-FP16-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-FP16-NEXT:    ldr h3, [x8, :lo12:.LCPI65_0]
 ; CHECK-GI-FP16-NEXT:    mov x11, #9223372036854775807 // =0x7fffffffffffffff
 ; CHECK-GI-FP16-NEXT:    mov h4, v0.h[2]
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h1
-; CHECK-GI-FP16-NEXT:    asr x9, x8, #63
-; CHECK-GI-FP16-NEXT:    fcvtzs x12, h3
-; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
-; CHECK-GI-FP16-NEXT:    csel x9, x10, x9, lt
-; CHECK-GI-FP16-NEXT:    fcmp h0, h2
-; CHECK-GI-FP16-NEXT:    csinv x8, x8, xzr, le
-; CHECK-GI-FP16-NEXT:    csel x9, x11, x9, gt
+; CHECK-GI-FP16-NEXT:    asr x8, x9, #63
+; CHECK-GI-FP16-NEXT:    fcvtzs x12, h2
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x9, lt
+; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h3
+; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h0
 ; CHECK-GI-FP16-NEXT:    mov h0, v0.h[3]
-; CHECK-GI-FP16-NEXT:    csel x0, xzr, x8, vs
-; CHECK-GI-FP16-NEXT:    csel x1, xzr, x9, vs
-; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    csel x0, xzr, x9, vs
+; CHECK-GI-FP16-NEXT:    csel x1, xzr, x8, vs
+; CHECK-GI-FP16-NEXT:    fcmp h2, h1
 ; CHECK-GI-FP16-NEXT:    asr x8, x12, #63
 ; CHECK-GI-FP16-NEXT:    csel x9, xzr, x12, lt
 ; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
-; CHECK-GI-FP16-NEXT:    fcmp h3, h2
+; CHECK-GI-FP16-NEXT:    fcmp h2, h3
 ; CHECK-GI-FP16-NEXT:    fcvtzs x12, h4
 ; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
 ; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
-; CHECK-GI-FP16-NEXT:    fcmp h3, h3
+; CHECK-GI-FP16-NEXT:    fcmp h2, h2
 ; CHECK-GI-FP16-NEXT:    csel x2, xzr, x9, vs
 ; CHECK-GI-FP16-NEXT:    csel x3, xzr, x8, vs
 ; CHECK-GI-FP16-NEXT:    fcmp h4, h1
 ; CHECK-GI-FP16-NEXT:    asr x8, x12, #63
 ; CHECK-GI-FP16-NEXT:    csel x9, xzr, x12, lt
 ; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
-; CHECK-GI-FP16-NEXT:    fcmp h4, h2
+; CHECK-GI-FP16-NEXT:    fcmp h4, h3
 ; CHECK-GI-FP16-NEXT:    fcvtzs x12, h0
 ; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
 ; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
@@ -3570,7 +3574,7 @@ define <4 x i128> @test_signed_v4f16_v4i128(<4 x half> %f) {
 ; CHECK-GI-FP16-NEXT:    asr x8, x12, #63
 ; CHECK-GI-FP16-NEXT:    csel x9, xzr, x12, lt
 ; CHECK-GI-FP16-NEXT:    csel x8, x10, x8, lt
-; CHECK-GI-FP16-NEXT:    fcmp h0, h2
+; CHECK-GI-FP16-NEXT:    fcmp h0, h3
 ; CHECK-GI-FP16-NEXT:    csinv x9, x9, xzr, le
 ; CHECK-GI-FP16-NEXT:    csel x8, x11, x8, gt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h0
@@ -4512,27 +4516,29 @@ define <8 x i128> @test_signed_v8f16_v8i128(<8 x half> %f) {
 ;
 ; CHECK-GI-FP16-LABEL: test_signed_v8f16_v8i128:
 ; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #4, lsl #8
-; CHECK-GI-FP16-NEXT:    fcvtzs x9, h0
-; CHECK-GI-FP16-NEXT:    mov x10, #-9223372036854775808 // =0x8000000000000000
-; CHECK-GI-FP16-NEXT:    mvni v2.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    adrp x9, .LCPI75_1
+; CHECK-GI-FP16-NEXT:    fcvtzs x11, h0
 ; CHECK-GI-FP16-NEXT:    mov h3, v0.h[1]
+; CHECK-GI-FP16-NEXT:    ldr h1, [x9, :lo12:.LCPI75_1]
+; CHECK-GI-FP16-NEXT:    adrp x9, .LCPI75_0
+; CHECK-GI-FP16-NEXT:    mov x10, #-9223372036854775808 // =0x8000000000000000
+; CHECK-GI-FP16-NEXT:    ldr h2, [x9, :lo12:.LCPI75_0]
 ; CHECK-GI-FP16-NEXT:    mov h4, v0.h[2]
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h1
-; CHECK-GI-FP16-NEXT:    asr x11, x9, #63
+; CHECK-GI-FP16-NEXT:    asr x9, x11, #63
 ; CHECK-GI-FP16-NEXT:    fcvtzs x13, h3
-; CHECK-GI-FP16-NEXT:    fcvtzs x15, h4
-; CHECK-GI-FP16-NEXT:    csel x12, xzr, x9, lt
-; CHECK-GI-FP16-NEXT:    csel x11, x10, x11, lt
+; CHECK-GI-FP16-NEXT:    csel x11, xzr, x11, lt
+; CHECK-GI-FP16-NEXT:    csel x12, x10, x9, lt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h2
 ; CHECK-GI-FP16-NEXT:    mov x9, #9223372036854775807 // =0x7fffffffffffffff
-; CHECK-GI-FP16-NEXT:    csinv x12, x12, xzr, le
-; CHECK-GI-FP16-NEXT:    csel x11, x9, x11, gt
+; CHECK-GI-FP16-NEXT:    fcvtzs x15, h4
+; CHECK-GI-FP16-NEXT:    csinv x11, x11, xzr, le
+; CHECK-GI-FP16-NEXT:    csel x14, x9, x12, gt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h0
-; CHECK-GI-FP16-NEXT:    asr x14, x13, #63
-; CHECK-GI-FP16-NEXT:    csel x12, xzr, x12, vs
-; CHECK-GI-FP16-NEXT:    csel x11, xzr, x11, vs
+; CHECK-GI-FP16-NEXT:    csel x12, xzr, x11, vs
+; CHECK-GI-FP16-NEXT:    csel x11, xzr, x14, vs
 ; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    asr x14, x13, #63
 ; CHECK-GI-FP16-NEXT:    csel x13, xzr, x13, lt
 ; CHECK-GI-FP16-NEXT:    csel x14, x10, x14, lt
 ; CHECK-GI-FP16-NEXT:    fcmp h3, h2
diff --git a/llvm/test/CodeGen/AArch64/fptoui-sat-scalar.ll b/llvm/test/CodeGen/AArch64/fptoui-sat-scalar.ll
index a503f54a33f6e1..40c9201b0f79a3 100644
--- a/llvm/test/CodeGen/AArch64/fptoui-sat-scalar.ll
+++ b/llvm/test/CodeGen/AArch64/fptoui-sat-scalar.ll
@@ -719,12 +719,13 @@ define i100 @test_unsigned_i100_f16(half %f) nounwind {
 ;
 ; CHECK-GI-FP16-LABEL: test_unsigned_i100_f16:
 ; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzu x8, h0
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
-; CHECK-GI-FP16-NEXT:    mov x9, #68719476735 // =0xfffffffff
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h0
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI28_0
 ; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
-; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI28_0]
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x9, lt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    mov x9, #68719476735 // =0xfffffffff
 ; CHECK-GI-FP16-NEXT:    csinv x0, x8, xzr, le
 ; CHECK-GI-FP16-NEXT:    csel x1, x9, xzr, gt
 ; CHECK-GI-FP16-NEXT:    ret
@@ -768,10 +769,11 @@ define i128 @test_unsigned_i128_f16(half %f) nounwind {
 ;
 ; CHECK-GI-FP16-LABEL: test_unsigned_i128_f16:
 ; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzu x8, h0
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h0
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI29_0
 ; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
-; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
+; CHECK-GI-FP16-NEXT:    ldr h1, [x8, :lo12:.LCPI29_0]
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x9, lt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h1
 ; CHECK-GI-FP16-NEXT:    csinv x0, x8, xzr, le
 ; CHECK-GI-FP16-NEXT:    csetm x1, gt
diff --git a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
index bc743cb31f1947..c2b106dcbd2aca 100644
--- a/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
+++ b/llvm/test/CodeGen/AArch64/fptoui-sat-vector.ll
@@ -2627,33 +2627,34 @@ define <4 x i100> @test_unsigned_v4f16_v4i100(<4 x half> %f) {
 ; CHECK-GI-FP16-LABEL: test_unsigned_v4f16_v4i100:
 ; CHECK-GI-FP16:       // %bb.0:
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    fcvtzu x8, h0
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
-; CHECK-GI-FP16-NEXT:    mov x9, #68719476735 // =0xfffffffff
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h0
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI64_0
 ; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    ldr h2, [x8, :lo12:.LCPI64_0]
 ; CHECK-GI-FP16-NEXT:    mov h3, v0.h[2]
-; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
-; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x9, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h2
+; CHECK-GI-FP16-NEXT:    mov x9, #68719476735 // =0xfffffffff
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h1
 ; CHECK-GI-FP16-NEXT:    mov h0, v0.h[3]
-; CHECK-GI-FP16-NEXT:    fcvtzu x10, h2
 ; CHECK-GI-FP16-NEXT:    csinv x0, x8, xzr, le
 ; CHECK-GI-FP16-NEXT:    csel x1, x9, xzr, gt
-; CHECK-GI-FP16-NEXT:    fcmp h2, #0.0
+; CHECK-GI-FP16-NEXT:    fcmp h1, #0.0
 ; CHECK-GI-FP16-NEXT:    csel x8, xzr, x10, lt
-; CHECK-GI-FP16-NEXT:    fcmp h2, h1
+; CHECK-GI-FP16-NEXT:    fcmp h1, h2
 ; CHECK-GI-FP16-NEXT:    fcvtzu x10, h3
 ; CHECK-GI-FP16-NEXT:    csinv x2, x8, xzr, le
 ; CHECK-GI-FP16-NEXT:    csel x3, x9, xzr, gt
 ; CHECK-GI-FP16-NEXT:    fcmp h3, #0.0
 ; CHECK-GI-FP16-NEXT:    csel x8, xzr, x10, lt
-; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    fcmp h3, h2
 ; CHECK-GI-FP16-NEXT:    fcvtzu x10, h0
 ; CHECK-GI-FP16-NEXT:    csinv x4, x8, xzr, le
 ; CHECK-GI-FP16-NEXT:    csel x5, x9, xzr, gt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
 ; CHECK-GI-FP16-NEXT:    csel x8, xzr, x10, lt
-; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    fcmp h0, h2
 ; CHECK-GI-FP16-NEXT:    csinv x6, x8, xzr, le
 ; CHECK-GI-FP16-NEXT:    csel x7, x9, xzr, gt
 ; CHECK-GI-FP16-NEXT:    ret
@@ -2782,32 +2783,33 @@ define <4 x i128> @test_unsigned_v4f16_v4i128(<4 x half> %f) {
 ; CHECK-GI-FP16-LABEL: test_unsigned_v4f16_v4i128:
 ; CHECK-GI-FP16:       // %bb.0:
 ; CHECK-GI-FP16-NEXT:    // kill: def $d0 killed $d0 def $q0
-; CHECK-GI-FP16-NEXT:    fcvtzu x8, h0
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h0
+; CHECK-GI-FP16-NEXT:    adrp x8, .LCPI65_0
 ; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
-; CHECK-GI-FP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-FP16-NEXT:    mov h1, v0.h[1]
+; CHECK-GI-FP16-NEXT:    ldr h2, [x8, :lo12:.LCPI65_0]
 ; CHECK-GI-FP16-NEXT:    mov h3, v0.h[2]
-; CHECK-GI-FP16-NEXT:    csel x8, xzr, x8, lt
-; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    csel x8, xzr, x9, lt
+; CHECK-GI-FP16-NEXT:    fcmp h0, h2
 ; CHECK-GI-FP16-NEXT:    mov h0, v0.h[3]
-; CHECK-GI-FP16-NEXT:    fcvtzu x9, h2
+; CHECK-GI-FP16-NEXT:    fcvtzu x9, h1
 ; CHECK-GI-FP16-NEXT:    csinv x0, x8, xzr, le
 ; CHECK-GI-FP16-NEXT:    csetm x1, gt
-; CHECK-GI-FP16-NEXT:    fcmp h2, #0.0
+; CHECK-GI-FP16-NEXT:    fcmp h1, #0.0
 ; CHECK-GI-FP16-NEXT:    csel x8, xzr, x9, lt
-; CHECK-GI-FP16-NEXT:    fcmp h2, h1
+; CHECK-GI-FP16-NEXT:    fcmp h1, h2
 ; CHECK-GI-FP16-NEXT:    fcvtzu x9, h3
 ; CHECK-GI-FP16-NEXT:    csinv x2, x8, xzr, le
 ; CHECK-GI-FP16-NEXT:    csetm x3, gt
 ; CHECK-GI-FP16-NEXT:    fcmp h3, #0.0
 ; CHECK-GI-FP16-NEXT:    csel x8, xzr, x9, lt
-; CHECK-GI-FP16-NEXT:    fcmp h3, h1
+; CHECK-GI-FP16-NEXT:    fcmp h3, h2
 ; CHECK-GI-FP16-NEXT:    fcvtzu x9, h0
 ; CHECK-GI-FP16-NEXT:    csinv x4, x8, xzr, le
 ; CHECK-GI-FP16-NEXT:    csetm x5, gt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
 ; CHECK-GI-FP16-NEXT:    csel x8, xzr, x9, lt
-; CHECK-GI-FP16-NEXT:    fcmp h0, h1
+; CHECK-GI-FP16-NEXT:    fcmp h0, h2
 ; CHECK-GI-FP16-NEXT:    csinv x6, x8, xzr, le
 ; CHECK-GI-FP16-NEXT:    csetm x7, gt
 ; CHECK-GI-FP16-NEXT:    ret
@@ -3573,12 +3575,13 @@ define <8 x i128> @test_unsigned_v8f16_v8i128(<8 x half> %f) {
 ;
 ; CHECK-GI-FP16-LABEL: test_unsigned_v8f16_v8i128:
 ; CHECK-GI-FP16:       // %bb.0:
-; CHECK-GI-FP16-NEXT:    fcvtzu x9, h0
-; CHECK-GI-FP16-NEXT:    mvni v1.4h, #132, lsl #8
+; CHECK-GI-FP16-NEXT:    fcvtzu x10, h0
+; CHECK-GI-FP16-NEXT:    adrp x9, .LCPI75_0
 ; CHECK-GI-FP16-NEXT:    fcmp h0, #0.0
 ; CHECK-GI-FP16-NEXT:    mov h2, v0.h[1]
+; CHECK-GI-FP16-NEXT:    ldr h1, [x9, :lo12:.LCPI75_0]
 ; CHECK-GI-FP16-NEXT:    mov h3, v0.h[2]
-; CHECK-GI-FP16-NEXT:    csel x9, xzr, x9, lt
+; CHECK-GI-FP16-NEXT:    csel x9, xzr, x10, lt
 ; CHECK-GI-FP16-NEXT:    fcmp h0, h1
 ; CHECK-GI-FP16-NEXT:    fcvtzu x10, h2
 ; CHECK-GI-FP16-NEXT:    fcvtzu x13, h3



More information about the llvm-branch-commits mailing list