[llvm] 57fe7f4 - [AMDGPU] NFC. Clean up check prefixes in fcmp test

Tue Apr 11 06:58:19 PDT 2023

Author: Joe Nash
Date: 2023-04-11T09:58:03-04:00
New Revision: 57fe7f450d7e2d565a984573174515b495562b77

URL: https://github.com/llvm/llvm-project/commit/57fe7f450d7e2d565a984573174515b495562b77
DIFF: https://github.com/llvm/llvm-project/commit/57fe7f450d7e2d565a984573174515b495562b77.diff

LOG: [AMDGPU] NFC. Clean up check prefixes in fcmp test

Fix the test introduced in D136592 which appeared to have a few check lines
containing an un-checked prefix "GISEL-GFX".
Also canonicalize the other prefixes to minimize churn if SDag and GISel
diverge.

Reviewed By: arsenm

Differential Revision: https://reviews.llvm.org/D147958

Added: 
    

Modified: 
    llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll

Removed: 
    


################################################################################
diff  --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
index 9b7fff3a7ebb4..b59bc60b51231 100644

--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.fcmp.w64.ll
@@ -1,11 +1,10 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -march=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX11 %s
-; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI %s
-; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9 %s
-
-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX11,GISEL-GFX11 %s
-; RUN: llc -global-isel -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,VI,GISEL-VI %s
-; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GCN,GFX9,GISEL-GFX9 %s
+; RUN: llc -march=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx1100 -mattr="+wavefrontsize64" -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL %s
+; RUN: llc -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=gfx900 -verify-machineinstrs < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
+; RUN: llc -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=VI-SDAG %s
+; RUN: llc -global-isel -march=amdgcn -mcpu=fiji -verify-machineinstrs < %s | FileCheck -check-prefixes=VI-GISEL %s
 
 declare i64 @llvm.amdgcn.fcmp.f32(float, float, i32) #0
 declare i64 @llvm.amdgcn.fcmp.f64(double, double, i32) #0
@@ -40,30 +39,31 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_with_fabs(ptr addrspace(1) %out, float
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_oeq_with_fabs:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s3
-; GISEL-VI-NEXT:    v_cmp_eq_f32_e64 s[2:3], s2, |v0|
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_oeq_with_fabs:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s3
-; SDAG-VI-NEXT:    v_cmp_eq_f32_e64 s[2:3], s2, |v0|
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_oeq_with_fabs:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s3
+; VI-SDAG-NEXT:    v_cmp_eq_f32_e64 s[2:3], s2, |v0|
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_oeq_with_fabs:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT:    v_cmp_eq_f32_e64 s[2:3], s2, |v0|
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %temp = call float @llvm.fabs.f32(float %a)
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float %temp, i32 1)
   store i64 %result, ptr addrspace(1) %out
@@ -96,30 +96,31 @@ define amdgpu_kernel void @v_fcmp_f32_oeq_both_operands_with_fabs(ptr addrspace(
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s3
-; GISEL-VI-NEXT:    v_cmp_eq_f32_e64 s[2:3], |s2|, |v0|
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s3
-; SDAG-VI-NEXT:    v_cmp_eq_f32_e64 s[2:3], |s2|, |v0|
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s3
+; VI-SDAG-NEXT:    v_cmp_eq_f32_e64 s[2:3], |s2|, |v0|
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_oeq_both_operands_with_fabs:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT:    v_cmp_eq_f32_e64 s[2:3], |s2|, |v0|
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %temp = call float @llvm.fabs.f32(float %a)
   %src_input = call float @llvm.fabs.f32(float %src)
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src_input, float %temp, i32 1)
@@ -132,50 +133,43 @@ define amdgpu_kernel void @v_fcmp_f32(ptr addrspace(1) %out, float %src) {
 ; SDAG-GFX:       ; %bb.0:
 ; SDAG-GFX-NEXT:    s_endpgm
 ;
-; SDAG-GFX11-LABEL: v_fcmp_f32:
-; SDAG-GFX11:       ; %bb.0:
-; SDAG-GFX11-NEXT:    s_endpgm
-;
-; SDAG-VI-LABEL: v_fcmp_f32:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_endpgm
-;
-; SDAG-GFX9-LABEL: v_fcmp_f32:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_endpgm
-;
-; GISEL-GFX11-LABEL: v_fcmp_f32:
-; GISEL-GFX11:       ; %bb.0:
-; GISEL-GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
-; GISEL-GFX11-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX11-NEXT:    global_store_b64 v0, v[0:1], s[0:1]
-; GISEL-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GISEL-GFX11-NEXT:    s_endpgm
-;
-; GISEL-VI-LABEL: v_fcmp_f32:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s1
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f32:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; GISEL-GFX-LABEL: v_fcmp_f32:
-; GISEL-GFX:       ; %bb.0:
-; GISEL-GFX-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-GFX-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-GFX-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
-; GISEL-GFX-NEXT:    s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f32:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_fcmp_f32:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_store_b64 v0, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f32:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f32:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f32:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 -1)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -210,32 +204,33 @@ define amdgpu_kernel void @v_fcmp_f32_oeq(ptr addrspace(1) %out, float %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_oeq:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_eq_f32_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_oeq:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_eq_f32_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_oeq:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_eq_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_oeq:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_eq_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 1)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -270,32 +265,33 @@ define amdgpu_kernel void @v_fcmp_f32_one(ptr addrspace(1) %out, float %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_one:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_neq_f32_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_one:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_neq_f32_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_one:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_neq_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_one:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_neq_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 6)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -330,32 +326,33 @@ define amdgpu_kernel void @v_fcmp_f32_ogt(ptr addrspace(1) %out, float %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_ogt:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_gt_f32_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_ogt:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_gt_f32_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_ogt:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_gt_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_ogt:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_gt_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 2)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -390,32 +387,33 @@ define amdgpu_kernel void @v_fcmp_f32_oge(ptr addrspace(1) %out, float %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_oge:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_ge_f32_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_oge:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_ge_f32_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_oge:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_ge_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_oge:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_ge_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 3)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -450,32 +448,33 @@ define amdgpu_kernel void @v_fcmp_f32_olt(ptr addrspace(1) %out, float %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_olt:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_lt_f32_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_olt:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_lt_f32_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_olt:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_lt_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_olt:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_lt_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 4)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -510,32 +509,33 @@ define amdgpu_kernel void @v_fcmp_f32_ole(ptr addrspace(1) %out, float %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_ole:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_le_f32_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_ole:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_le_f32_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_ole:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_le_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_ole:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_le_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 5)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -571,32 +571,33 @@ define amdgpu_kernel void @v_fcmp_f32_ueq(ptr addrspace(1) %out, float %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_ueq:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_nlg_f32_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_ueq:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_nlg_f32_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_ueq:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_nlg_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_ueq:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_nlg_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 9)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -631,32 +632,33 @@ define amdgpu_kernel void @v_fcmp_f32_une(ptr addrspace(1) %out, float %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_une:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_neq_f32_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_une:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_neq_f32_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_une:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_neq_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_une:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_neq_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 14)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -691,32 +693,33 @@ define amdgpu_kernel void @v_fcmp_f32_ugt(ptr addrspace(1) %out, float %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_ugt:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_nle_f32_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_ugt:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_nle_f32_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_ugt:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_nle_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_ugt:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_nle_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 10)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -751,32 +754,33 @@ define amdgpu_kernel void @v_fcmp_f32_uge(ptr addrspace(1) %out, float %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_uge:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_nlt_f32_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_uge:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_nlt_f32_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_uge:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_nlt_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_uge:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_nlt_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 11)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -811,32 +815,33 @@ define amdgpu_kernel void @v_fcmp_f32_ult(ptr addrspace(1) %out, float %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_ult:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_nge_f32_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_ult:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_nge_f32_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_ult:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_nge_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_ult:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_nge_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 12)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -871,32 +876,33 @@ define amdgpu_kernel void @v_fcmp_f32_ule(ptr addrspace(1) %out, float %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f32_ule:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_ngt_f32_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f32_ule:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x42c80000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_ngt_f32_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f32_ule:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_ngt_f32_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f32_ule:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x42c80000
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_ngt_f32_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f32(float %src, float 100.00, i32 13)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -918,60 +924,62 @@ define amdgpu_kernel void @v_fcmp_f64_oeq(ptr addrspace(1) %out, double %src) {
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f64_oeq:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_mov_b32 s4, 0
-; GISEL-VI-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_eq_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f64_oeq:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    s_mov_b32 s4, 0
-; GISEL-GFX9-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    v_cmp_eq_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f64_oeq:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_eq_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
-; SDAG-GFX9-LABEL: v_fcmp_f64_oeq:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT:    v_cmp_eq_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v2, s3
-; SDAG-GFX9-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
-; SDAG-GFX9-NEXT:    s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f64_oeq:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_eq_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_oeq:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_eq_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_oeq:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_eq_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_oeq:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_eq_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 1)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -993,60 +1001,62 @@ define amdgpu_kernel void @v_fcmp_f64_one(ptr addrspace(1) %out, double %src) {
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f64_one:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_mov_b32 s4, 0
-; GISEL-VI-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f64_one:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    s_mov_b32 s4, 0
-; GISEL-GFX9-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f64_one:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
-; SDAG-GFX9-LABEL: v_fcmp_f64_one:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v2, s3
-; SDAG-GFX9-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
-; SDAG-GFX9-NEXT:    s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f64_one:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_one:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_one:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_one:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 6)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -1068,60 +1078,62 @@ define amdgpu_kernel void @v_fcmp_f64_ogt(ptr addrspace(1) %out, double %src) {
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f64_ogt:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_mov_b32 s4, 0
-; GISEL-VI-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_gt_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f64_ogt:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    s_mov_b32 s4, 0
-; GISEL-GFX9-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    v_cmp_gt_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f64_ogt:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_gt_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
-; SDAG-GFX9-LABEL: v_fcmp_f64_ogt:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT:    v_cmp_gt_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v2, s3
-; SDAG-GFX9-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
-; SDAG-GFX9-NEXT:    s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f64_ogt:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_gt_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_ogt:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_gt_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_ogt:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_gt_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_ogt:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_gt_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 2)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -1143,60 +1155,62 @@ define amdgpu_kernel void @v_fcmp_f64_oge(ptr addrspace(1) %out, double %src) {
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f64_oge:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_mov_b32 s4, 0
-; GISEL-VI-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_ge_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f64_oge:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    s_mov_b32 s4, 0
-; GISEL-GFX9-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    v_cmp_ge_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f64_oge:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_ge_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
-; SDAG-GFX9-LABEL: v_fcmp_f64_oge:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT:    v_cmp_ge_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v2, s3
-; SDAG-GFX9-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
-; SDAG-GFX9-NEXT:    s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f64_oge:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_ge_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_oge:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_ge_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_oge:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_ge_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_oge:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_ge_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 3)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -1218,60 +1232,62 @@ define amdgpu_kernel void @v_fcmp_f64_olt(ptr addrspace(1) %out, double %src) {
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f64_olt:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_mov_b32 s4, 0
-; GISEL-VI-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_lt_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f64_olt:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    s_mov_b32 s4, 0
-; GISEL-GFX9-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    v_cmp_lt_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f64_olt:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_lt_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
-; SDAG-GFX9-LABEL: v_fcmp_f64_olt:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT:    v_cmp_lt_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v2, s3
-; SDAG-GFX9-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
-; SDAG-GFX9-NEXT:    s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f64_olt:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_lt_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_olt:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_lt_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_olt:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_lt_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_olt:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_lt_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 4)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -1293,60 +1309,62 @@ define amdgpu_kernel void @v_fcmp_f64_ole(ptr addrspace(1) %out, double %src) {
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f64_ole:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_mov_b32 s4, 0
-; GISEL-VI-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_le_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f64_ole:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    s_mov_b32 s4, 0
-; GISEL-GFX9-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    v_cmp_le_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f64_ole:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_le_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
-; SDAG-GFX9-LABEL: v_fcmp_f64_ole:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT:    v_cmp_le_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v2, s3
-; SDAG-GFX9-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
-; SDAG-GFX9-NEXT:    s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f64_ole:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_le_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_ole:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_le_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_ole:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_le_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_ole:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_le_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 5)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -1368,60 +1386,62 @@ define amdgpu_kernel void @v_fcmp_f64_ueq(ptr addrspace(1) %out, double %src) {
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f64_ueq:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_mov_b32 s4, 0
-; GISEL-VI-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_nlg_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f64_ueq:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    s_mov_b32 s4, 0
-; GISEL-GFX9-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    v_cmp_nlg_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f64_ueq:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_nlg_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
-; SDAG-GFX9-LABEL: v_fcmp_f64_ueq:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT:    v_cmp_nlg_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v2, s3
-; SDAG-GFX9-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
-; SDAG-GFX9-NEXT:    s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f64_ueq:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_nlg_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_ueq:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_nlg_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_ueq:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_nlg_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_ueq:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_nlg_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 9)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -1443,60 +1463,62 @@ define amdgpu_kernel void @v_fcmp_f64_une(ptr addrspace(1) %out, double %src) {
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f64_une:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_mov_b32 s4, 0
-; GISEL-VI-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f64_une:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    s_mov_b32 s4, 0
-; GISEL-GFX9-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f64_une:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
-; SDAG-GFX9-LABEL: v_fcmp_f64_une:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v2, s3
-; SDAG-GFX9-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
-; SDAG-GFX9-NEXT:    s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f64_une:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_une:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_une:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_une:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_neq_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 14)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -1518,60 +1540,62 @@ define amdgpu_kernel void @v_fcmp_f64_ugt(ptr addrspace(1) %out, double %src) {
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f64_ugt:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_mov_b32 s4, 0
-; GISEL-VI-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_nle_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f64_ugt:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    s_mov_b32 s4, 0
-; GISEL-GFX9-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    v_cmp_nle_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f64_ugt:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_nle_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
-; SDAG-GFX9-LABEL: v_fcmp_f64_ugt:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT:    v_cmp_nle_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v2, s3
-; SDAG-GFX9-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
-; SDAG-GFX9-NEXT:    s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f64_ugt:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_nle_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_ugt:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_nle_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_ugt:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_nle_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_ugt:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_nle_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 10)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -1593,60 +1617,62 @@ define amdgpu_kernel void @v_fcmp_f64_uge(ptr addrspace(1) %out, double %src) {
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f64_uge:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_mov_b32 s4, 0
-; GISEL-VI-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_nlt_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f64_uge:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    s_mov_b32 s4, 0
-; GISEL-GFX9-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    v_cmp_nlt_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f64_uge:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_nlt_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
-; SDAG-GFX9-LABEL: v_fcmp_f64_uge:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT:    v_cmp_nlt_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v2, s3
-; SDAG-GFX9-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
-; SDAG-GFX9-NEXT:    s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f64_uge:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_nlt_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_uge:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_nlt_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_uge:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_nlt_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_uge:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_nlt_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 11)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -1668,60 +1694,62 @@ define amdgpu_kernel void @v_fcmp_f64_ult(ptr addrspace(1) %out, double %src) {
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f64_ult:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_mov_b32 s4, 0
-; GISEL-VI-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_nge_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f64_ult:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    s_mov_b32 s4, 0
-; GISEL-GFX9-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    v_cmp_nge_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f64_ult:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_nge_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
-; SDAG-GFX9-LABEL: v_fcmp_f64_ult:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT:    v_cmp_nge_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v2, s3
-; SDAG-GFX9-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
-; SDAG-GFX9-NEXT:    s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f64_ult:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_nge_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_ult:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_nge_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_ult:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_nge_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_ult:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_nge_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 12)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -1743,60 +1771,62 @@ define amdgpu_kernel void @v_fcmp_f64_ule(ptr addrspace(1) %out, double %src) {
 ; GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
 ; GFX11-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f64_ule:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_mov_b32 s4, 0
-; GISEL-VI-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_ngt_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f64_ule:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    s_mov_b32 s4, 0
-; GISEL-GFX9-NEXT:    s_mov_b32 s5, 0x40590000
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s4
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s5
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    v_cmp_ngt_f64_e64 s[2:3], s[2:3], v[0:1]
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v2, 0
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f64_ule:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_ngt_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
-; SDAG-GFX9-LABEL: v_fcmp_f64_ule:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, 0x40590000
-; SDAG-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-GFX9-NEXT:    v_cmp_ngt_f64_e64 s[2:3], s[2:3], v[0:1]
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v1, s2
-; SDAG-GFX9-NEXT:    v_mov_b32_e32 v2, s3
-; SDAG-GFX9-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
-; SDAG-GFX9-NEXT:    s_endpgm
+; GFX9-SDAG-LABEL: v_fcmp_f64_ule:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; GFX9-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-SDAG-NEXT:    v_cmp_ngt_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v1, s2
+; GFX9-SDAG-NEXT:    v_mov_b32_e32 v2, s3
+; GFX9-SDAG-NEXT:    global_store_dwordx2 v0, v[1:2], s[0:1]
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f64_ule:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    s_mov_b32 s4, 0
+; GFX9-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    v_cmp_ngt_f64_e64 s[2:3], s[2:3], v[0:1]
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v2, 0
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v2, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f64_ule:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, 0x40590000
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_ngt_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f64_ule:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx4 s[0:3], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_mov_b32 s4, 0
+; VI-GISEL-NEXT:    s_mov_b32 s5, 0x40590000
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s4
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s5
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_ngt_f64_e64 s[2:3], s[2:3], v[0:1]
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f64(double %src, double 100.00, i32 13)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -1835,34 +1865,35 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_with_fabs(ptr addrspace(1) %out, half
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_oeq_with_fabs:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    s_lshr_b32 s3, s2, 16
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s3
-; GISEL-VI-NEXT:    v_cmp_eq_f16_e64 s[2:3], s2, |v0|
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_oeq_with_fabs:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    s_lshr_b32 s3, s2, 16
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s3
-; SDAG-VI-NEXT:    v_cmp_eq_f16_e64 s[2:3], s2, |v0|
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_oeq_with_fabs:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    s_lshr_b32 s3, s2, 16
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s3
+; VI-SDAG-NEXT:    v_cmp_eq_f16_e64 s[2:3], s2, |v0|
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_oeq_with_fabs:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    s_lshr_b32 s3, s2, 16
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT:    v_cmp_eq_f16_e64 s[2:3], s2, |v0|
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %temp = call half @llvm.fabs.f16(half %a)
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half %temp, i32 1)
   store i64 %result, ptr addrspace(1) %out
@@ -1902,34 +1933,35 @@ define amdgpu_kernel void @v_fcmp_f16_oeq_both_operands_with_fabs(ptr addrspace(
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    s_lshr_b32 s3, s2, 16
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s3
-; GISEL-VI-NEXT:    v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    s_lshr_b32 s3, s2, 16
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s3
-; SDAG-VI-NEXT:    v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    s_lshr_b32 s3, s2, 16
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s3
+; VI-SDAG-NEXT:    v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_oeq_both_operands_with_fabs:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    s_lshr_b32 s3, s2, 16
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s3
+; VI-GISEL-NEXT:    v_cmp_eq_f16_e64 s[2:3], |s2|, |v0|
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %temp = call half @llvm.fabs.f16(half %a)
   %src_input = call half @llvm.fabs.f16(half %src)
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src_input, half %temp, i32 1)
@@ -1942,50 +1974,43 @@ define amdgpu_kernel void @v_fcmp_f16(ptr addrspace(1) %out, half %src) {
 ; SDAG-GFX:       ; %bb.0:
 ; SDAG-GFX-NEXT:    s_endpgm
 ;
-; SDAG-GFX11-LABEL: v_fcmp_f16:
-; SDAG-GFX11:       ; %bb.0:
-; SDAG-GFX11-NEXT:    s_endpgm
-;
-; SDAG-VI-LABEL: v_fcmp_f16:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_endpgm
-;
-; SDAG-GFX9-LABEL: v_fcmp_f16:
-; SDAG-GFX9:       ; %bb.0:
-; SDAG-GFX9-NEXT:    s_endpgm
-;
-; GISEL-GFX11-LABEL: v_fcmp_f16:
-; GISEL-GFX11:       ; %bb.0:
-; GISEL-GFX11-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
-; GISEL-GFX11-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-GFX11-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX11-NEXT:    global_store_b64 v0, v[0:1], s[0:1]
-; GISEL-GFX11-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
-; GISEL-GFX11-NEXT:    s_endpgm
-;
-; GISEL-VI-LABEL: v_fcmp_f16:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s1
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-;
-; GISEL-GFX9-LABEL: v_fcmp_f16:
-; GISEL-GFX9:       ; %bb.0:
-; GISEL-GFX9-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-GFX9-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-GFX9-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX9-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
-; GISEL-GFX9-NEXT:    s_endpgm
-; GISEL-GFX-LABEL: v_fcmp_f16:
-; GISEL-GFX:       ; %bb.0:
-; GISEL-GFX-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-GFX-NEXT:    v_mov_b32_e32 v0, 0
-; GISEL-GFX-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-GFX-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
-; GISEL-GFX-NEXT:    s_endpgm
+; GFX11-SDAG-LABEL: v_fcmp_f16:
+; GFX11-SDAG:       ; %bb.0:
+; GFX11-SDAG-NEXT:    s_endpgm
+;
+; GFX11-GISEL-LABEL: v_fcmp_f16:
+; GFX11-GISEL:       ; %bb.0:
+; GFX11-GISEL-NEXT:    s_load_b64 s[0:1], s[0:1], 0x24
+; GFX11-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX11-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX11-GISEL-NEXT:    global_store_b64 v0, v[0:1], s[0:1]
+; GFX11-GISEL-NEXT:    s_sendmsg sendmsg(MSG_DEALLOC_VGPRS)
+; GFX11-GISEL-NEXT:    s_endpgm
+;
+; GFX9-SDAG-LABEL: v_fcmp_f16:
+; GFX9-SDAG:       ; %bb.0:
+; GFX9-SDAG-NEXT:    s_endpgm
+;
+; GFX9-GISEL-LABEL: v_fcmp_f16:
+; GFX9-GISEL:       ; %bb.0:
+; GFX9-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; GFX9-GISEL-NEXT:    v_mov_b32_e32 v0, 0
+; GFX9-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; GFX9-GISEL-NEXT:    global_store_dwordx2 v0, v[0:1], s[0:1]
+; GFX9-GISEL-NEXT:    s_endpgm
+;
+; VI-SDAG-LABEL: v_fcmp_f16:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s1
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[0:1], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 -1)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -2021,32 +2046,33 @@ define amdgpu_kernel void @v_fcmp_f16_oeq(ptr addrspace(1) %out, half %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_oeq:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_eq_f16_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_oeq:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_eq_f16_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_oeq:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_eq_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_oeq:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_eq_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 1)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -2082,32 +2108,33 @@ define amdgpu_kernel void @v_fcmp_f16_one(ptr addrspace(1) %out, half %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_one:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_neq_f16_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_one:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_neq_f16_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_one:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_neq_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_one:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_neq_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 6)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -2143,32 +2170,33 @@ define amdgpu_kernel void @v_fcmp_f16_ogt(ptr addrspace(1) %out, half %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_ogt:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_gt_f16_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_ogt:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_gt_f16_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_ogt:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_gt_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_ogt:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_gt_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 2)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -2204,32 +2232,33 @@ define amdgpu_kernel void @v_fcmp_f16_oge(ptr addrspace(1) %out, half %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_oge:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_ge_f16_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_oge:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_ge_f16_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_oge:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_ge_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_oge:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_ge_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 3)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -2265,32 +2294,33 @@ define amdgpu_kernel void @v_fcmp_f16_olt(ptr addrspace(1) %out, half %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_olt:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_lt_f16_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_olt:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_lt_f16_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_olt:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_lt_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_olt:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_lt_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 4)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -2326,32 +2356,33 @@ define amdgpu_kernel void @v_fcmp_f16_ole(ptr addrspace(1) %out, half %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_ole:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_le_f16_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_ole:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_le_f16_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_ole:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_le_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_ole:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_le_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 5)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -2387,32 +2418,33 @@ define amdgpu_kernel void @v_fcmp_f16_ueq(ptr addrspace(1) %out, half %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_ueq:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_nlg_f16_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_ueq:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_nlg_f16_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_ueq:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_nlg_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_ueq:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_nlg_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 9)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -2448,32 +2480,33 @@ define amdgpu_kernel void @v_fcmp_f16_une(ptr addrspace(1) %out, half %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_une:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_neq_f16_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_une:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_neq_f16_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_une:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_neq_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_une:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_neq_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 14)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -2509,32 +2542,33 @@ define amdgpu_kernel void @v_fcmp_f16_ugt(ptr addrspace(1) %out, half %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_ugt:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_nle_f16_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_ugt:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_nle_f16_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_ugt:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_nle_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_ugt:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_nle_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 10)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -2570,32 +2604,33 @@ define amdgpu_kernel void @v_fcmp_f16_uge(ptr addrspace(1) %out, half %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_uge:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_nlt_f16_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_uge:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_nlt_f16_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_uge:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_nlt_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_uge:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_nlt_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 11)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -2631,32 +2666,33 @@ define amdgpu_kernel void @v_fcmp_f16_ult(ptr addrspace(1) %out, half %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_ult:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_nge_f16_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_ult:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_nge_f16_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_ult:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_nge_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_ult:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_nge_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 12)
   store i64 %result, ptr addrspace(1) %out
   ret void
@@ -2692,38 +2728,36 @@ define amdgpu_kernel void @v_fcmp_f16_ule(ptr addrspace(1) %out, half %src) {
 ; GFX9-NEXT:    global_store_dwordx2 v2, v[0:1], s[2:3]
 ; GFX9-NEXT:    s_endpgm
 ;
-; GISEL-VI-LABEL: v_fcmp_f16_ule:
-; GISEL-VI:       ; %bb.0:
-; GISEL-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; GISEL-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; GISEL-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; GISEL-VI-NEXT:    v_cmp_ngt_f16_e64 s[2:3], s2, v0
-; GISEL-VI-NEXT:    v_mov_b32_e32 v0, s2
-; GISEL-VI-NEXT:    v_mov_b32_e32 v3, s1
-; GISEL-VI-NEXT:    v_mov_b32_e32 v1, s3
-; GISEL-VI-NEXT:    v_mov_b32_e32 v2, s0
-; GISEL-VI-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
-; GISEL-VI-NEXT:    s_endpgm
-; SDAG-VI-LABEL: v_fcmp_f16_ule:
-; SDAG-VI:       ; %bb.0:
-; SDAG-VI-NEXT:    s_load_dword s2, s[0:1], 0x2c
-; SDAG-VI-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, 0x5640
-; SDAG-VI-NEXT:    s_waitcnt lgkmcnt(0)
-; SDAG-VI-NEXT:    v_cmp_ngt_f16_e64 s[2:3], s2, v0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v0, s0
-; SDAG-VI-NEXT:    v_mov_b32_e32 v2, s2
-; SDAG-VI-NEXT:    v_mov_b32_e32 v1, s1
-; SDAG-VI-NEXT:    v_mov_b32_e32 v3, s3
-; SDAG-VI-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
-; SDAG-VI-NEXT:    s_endpgm
+; VI-SDAG-LABEL: v_fcmp_f16_ule:
+; VI-SDAG:       ; %bb.0:
+; VI-SDAG-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-SDAG-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-SDAG-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-SDAG-NEXT:    v_cmp_ngt_f16_e64 s[2:3], s2, v0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v0, s0
+; VI-SDAG-NEXT:    v_mov_b32_e32 v2, s2
+; VI-SDAG-NEXT:    v_mov_b32_e32 v1, s1
+; VI-SDAG-NEXT:    v_mov_b32_e32 v3, s3
+; VI-SDAG-NEXT:    flat_store_dwordx2 v[0:1], v[2:3]
+; VI-SDAG-NEXT:    s_endpgm
+;
+; VI-GISEL-LABEL: v_fcmp_f16_ule:
+; VI-GISEL:       ; %bb.0:
+; VI-GISEL-NEXT:    s_load_dword s2, s[0:1], 0x2c
+; VI-GISEL-NEXT:    s_load_dwordx2 s[0:1], s[0:1], 0x24
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, 0x5640
+; VI-GISEL-NEXT:    s_waitcnt lgkmcnt(0)
+; VI-GISEL-NEXT:    v_cmp_ngt_f16_e64 s[2:3], s2, v0
+; VI-GISEL-NEXT:    v_mov_b32_e32 v0, s2
+; VI-GISEL-NEXT:    v_mov_b32_e32 v3, s1
+; VI-GISEL-NEXT:    v_mov_b32_e32 v1, s3
+; VI-GISEL-NEXT:    v_mov_b32_e32 v2, s0
+; VI-GISEL-NEXT:    flat_store_dwordx2 v[2:3], v[0:1]
+; VI-GISEL-NEXT:    s_endpgm
   %result = call i64 @llvm.amdgcn.fcmp.f16(half %src, half 100.00, i32 13)
   store i64 %result, ptr addrspace(1) %out
   ret void
 }
 
 attributes #0 = { nounwind readnone convergent }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GCN: {{.*}}
-; VI: {{.*}}


        
