[llvm] [AMDGPU] Enable scalar subword loads for plain 16-bit loads (PR #225097)
via llvm-commits
llvm-commits at lists.llvm.org
Wed Sep 23 03:24:29 PDT 2026
https://github.com/LU-JOHN updated https://github.com/llvm/llvm-project/pull/225097
>From 63082e4224a05e3a84f8b0c6879b9f52f5c006ce Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Mon, 21 Sep 2026 09:30:03 -0500
Subject: [PATCH 1/2] Enable scalar subword loads for plain 16-bit loads
Signed-off-by: John Lu <John.Lu at amd.com>
---
llvm/lib/Target/AMDGPU/SMInstructions.td | 21 +-
.../test/CodeGen/AMDGPU/fcanonicalize.bf16.ll | 50 +-
.../CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll | 42 +-
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll | 42 +-
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 819 +++++++-----------
llvm/test/CodeGen/AMDGPU/load-constant-i16.ll | 217 ++---
llvm/test/CodeGen/AMDGPU/load-constant-i8.ll | 238 ++---
llvm/test/CodeGen/AMDGPU/min.ll | 22 +-
.../AMDGPU/preload-implicit-kernargs.ll | 38 +-
9 files changed, 582 insertions(+), 907 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index 19aeafe9b30ccc..0f155fd898e981 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -980,7 +980,7 @@ multiclass SMLoad_Pattern <string Instr, ValueType vt, bit immci = true> {
}
}
-multiclass ScalarLoadWithExtensionPat <string Instr, SDPatternOperator node, ValueType vt> {
+multiclass ScalarSubwordLoadPat <string Instr, SDPatternOperator node, ValueType vt> {
// 1. IMM offset
def : GCNPat <
(node (SMRDImm i64:$sbase, i32:$offset)),
@@ -1041,12 +1041,19 @@ multiclass ScalarBufferLoadIntrinsicPat <SDPatternOperator name, string Instr> {
// selector to prefer those.
let AddedComplexity = 100 in {
-defm : ScalarLoadWithExtensionPat <"S_LOAD_U8", smrd_extloadi8, i32>;
-defm : ScalarLoadWithExtensionPat <"S_LOAD_U8", smrd_zextloadi8, i32>;
-defm : ScalarLoadWithExtensionPat <"S_LOAD_I8", smrd_sextloadi8, i32>;
-defm : ScalarLoadWithExtensionPat <"S_LOAD_U16", smrd_extloadi16, i32>;
-defm : ScalarLoadWithExtensionPat <"S_LOAD_U16", smrd_zextloadi16, i32>;
-defm : ScalarLoadWithExtensionPat <"S_LOAD_I16", smrd_sextloadi16, i32>;
+// Extending subword loads
+defm : ScalarSubwordLoadPat <"S_LOAD_U8", smrd_extloadi8, i32>;
+defm : ScalarSubwordLoadPat <"S_LOAD_U8", smrd_zextloadi8, i32>;
+defm : ScalarSubwordLoadPat <"S_LOAD_I8", smrd_sextloadi8, i32>;
+defm : ScalarSubwordLoadPat <"S_LOAD_U16", smrd_extloadi16, i32>;
+defm : ScalarSubwordLoadPat <"S_LOAD_U16", smrd_zextloadi16, i32>;
+defm : ScalarSubwordLoadPat <"S_LOAD_I16", smrd_sextloadi16, i32>;
+
+// Non-extending subword loads
+foreach vt = Reg16Types.types in {
+ defm : ScalarSubwordLoadPat <"S_LOAD_U16", smrd_load, vt>;
+}
+
defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_byte, "S_BUFFER_LOAD_I8">;
defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_ubyte, "S_BUFFER_LOAD_U8">;
defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_short, "S_BUFFER_LOAD_I16">;
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
index f559943070dd77..6ad79905071133 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
@@ -120,11 +120,11 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_bf16(ptr addrspace(1) %o
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v1, 0x7fff, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_and_b32 s2, s2, 0x7fff
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, s2 op_sel_hi:[0,1]
; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
@@ -145,11 +145,11 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_bf16(ptr addrspace(
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_or_b32_e32 v1, 0x8000, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_bitset1_b32 s2, 15
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, s2 op_sel_hi:[0,1]
; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
@@ -170,11 +170,11 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_bf16(ptr addrspace(1) %o
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v1, 0x8000, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_xor_b32 s2, s2, 0x8000
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, s2 op_sel_hi:[0,1]
; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
@@ -194,11 +194,11 @@ define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_var_bf16(ptr ad
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v1, 0x8000, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_xor_b32 s2, s2, 0x8000
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, s2 op_sel_hi:[0,1]
; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
@@ -218,11 +218,11 @@ define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_fabs_var_bf16(p
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_or_b32_e32 v1, 0x8000, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_bitset1_b32 s2, 15
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, s2 op_sel_hi:[0,1]
; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll
index 73f8df5ed65153..73304ba2b319df 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll
@@ -65,37 +65,31 @@ define amdgpu_kernel void @rcp_f16(
; GFX12-TRUE16-LABEL: rcp_f16:
; GFX12-TRUE16: ; %bb.0: ; %entry
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX12-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX12-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_rcp_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_s_rcp_f16 s2, s2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-TRUE16-NEXT: s_endpgm
;
; GFX12-FAKE16-LABEL: rcp_f16:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX12-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX12-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX12-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX12-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_rcp_f16_e32 v0, v0
-; GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_s_rcp_f16 s2, s2
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
index e5979d8c99b606..fd36da8c635bf8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
@@ -70,37 +70,31 @@ define amdgpu_kernel void @rsq_f16(
; SDAG-GFX12-TRUE16-LABEL: rsq_f16:
; SDAG-GFX12-TRUE16: ; %bb.0: ; %entry
; SDAG-GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s6, -1
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s10, s6
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s11, s7
; SDAG-GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; SDAG-GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
-; SDAG-GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; SDAG-GFX12-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
-; SDAG-GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; SDAG-GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; SDAG-GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT: v_s_rsq_f16 s2, s2
+; SDAG-GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; SDAG-GFX12-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; SDAG-GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s2, -1
+; SDAG-GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; SDAG-GFX12-TRUE16-NEXT: s_endpgm
;
; SDAG-GFX12-FAKE16-LABEL: rsq_f16:
; SDAG-GFX12-FAKE16: ; %bb.0: ; %entry
; SDAG-GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s6, -1
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s10, s6
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s11, s7
; SDAG-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s8, s2
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s9, s3
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s4, s0
-; SDAG-GFX12-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s5, s1
-; SDAG-GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; SDAG-GFX12-FAKE16-NEXT: v_rsq_f16_e32 v0, v0
-; SDAG-GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; SDAG-GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; SDAG-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT: v_s_rsq_f16 s2, s2
+; SDAG-GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; SDAG-GFX12-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; SDAG-GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s2, -1
+; SDAG-GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; SDAG-GFX12-FAKE16-NEXT: s_endpgm
;
; GISEL-GCN-LABEL: rsq_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index 34910b6f180e7c..17bb36e3d943e4 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -539,34 +539,50 @@ define amdgpu_kernel void @constant_load_v16i1(ptr addrspace(1) %out, ptr addrsp
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX12-TRUE16-NEXT: s_endpgm
;
; GFX12-FAKE16-LABEL: constant_load_v16i1:
; GFX12-FAKE16: ; %bb.0:
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX12-FAKE16-NEXT: s_endpgm
;
-; GFX1250-LABEL: constant_load_v16i1:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[2:3] nv
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
-; GFX1250-NEXT: s_endpgm
+; GFX1250-TRUE16-LABEL: constant_load_v16i1:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
+;
+; GFX1250-FAKE16-LABEL: constant_load_v16i1:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1250-FAKE16-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
store <16 x i1> %load, ptr addrspace(1) %out
ret void
@@ -2218,86 +2234,44 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; EG-NEXT: ADD_INT * T14.X, T11.X, literal.x,
; EG-NEXT: 12(1.681558e-44), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_zextload_v16i1_to_v16i32:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v16, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v16, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10003
-; GFX12-TRUE16-NEXT: s_bfe_u32 s5, s2, 0x10001
-; GFX12-TRUE16-NEXT: s_bfe_u32 s6, s2, 0x10007
-; GFX12-TRUE16-NEXT: s_bfe_u32 s7, s2, 0x10009
-; GFX12-TRUE16-NEXT: s_bfe_u32 s8, s2, 0x1000d
-; GFX12-TRUE16-NEXT: s_and_b32 s9, s2, 1
-; GFX12-TRUE16-NEXT: s_bfe_u32 s10, s2, 0x1000a
-; GFX12-TRUE16-NEXT: s_bfe_u32 s2, s2, 0x1000c
-; GFX12-TRUE16-NEXT: s_bfe_u32 s11, s3, 0x10005
-; GFX12-TRUE16-NEXT: s_bfe_u32 s12, s3, 0x1000b
-; GFX12-TRUE16-NEXT: s_lshr_b32 s13, s3, 15
-; GFX12-TRUE16-NEXT: s_bfe_u32 s14, s3, 0x10002
-; GFX12-TRUE16-NEXT: s_bfe_u32 s15, s3, 0x10006
-; GFX12-TRUE16-NEXT: s_bfe_u32 s16, s3, 0x10004
-; GFX12-TRUE16-NEXT: s_bfe_u32 s17, s3, 0x10008
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s3, 0x1000e
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v2, s3
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v8, s16
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s10
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v15, s4 :: v_dual_mov_b32 v4, s17
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v10, s15
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v11, s6 :: v_dual_mov_b32 v12, s9
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v14, s14
-; GFX12-TRUE16-NEXT: s_clause 0x3
-; GFX12-TRUE16-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
-; GFX12-TRUE16-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
-; GFX12-TRUE16-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
-; GFX12-TRUE16-NEXT: global_store_b128 v16, v[12:15], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_v16i1_to_v16i32:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v16, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v16, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-FAKE16-NEXT: s_and_b32 s6, 0xffff, s2
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x10001
-; GFX12-FAKE16-NEXT: s_bfe_u32 s5, s2, 0x10007
-; GFX12-FAKE16-NEXT: s_bfe_u32 s7, s2, 0x10009
-; GFX12-FAKE16-NEXT: s_bfe_u32 s8, s2, 0x1000d
-; GFX12-FAKE16-NEXT: s_and_b32 s9, s2, 1
-; GFX12-FAKE16-NEXT: s_bfe_u32 s10, s2, 0x1000a
-; GFX12-FAKE16-NEXT: s_bfe_u32 s2, s2, 0x1000c
-; GFX12-FAKE16-NEXT: s_bfe_u32 s11, s6, 0x10005
-; GFX12-FAKE16-NEXT: s_bfe_u32 s12, s6, 0x1000b
-; GFX12-FAKE16-NEXT: s_lshr_b32 s13, s6, 15
-; GFX12-FAKE16-NEXT: s_bfe_u32 s14, s6, 0x10002
-; GFX12-FAKE16-NEXT: s_bfe_u32 s15, s6, 0x10006
-; GFX12-FAKE16-NEXT: s_bfe_u32 s16, s6, 0x10004
-; GFX12-FAKE16-NEXT: s_bfe_u32 s17, s6, 0x10008
-; GFX12-FAKE16-NEXT: s_bfe_u32 s6, s6, 0x1000e
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v13, s4 :: v_dual_mov_b32 v2, s6
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v8, s16
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s10
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v4, s17
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v10, s15
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v12, s9
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v14, s14
-; GFX12-FAKE16-NEXT: s_clause 0x3
-; GFX12-FAKE16-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
-; GFX12-FAKE16-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
-; GFX12-FAKE16-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
-; GFX12-FAKE16-NEXT: global_store_b128 v16, v[12:15], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_v16i1_to_v16i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_and_b32 s6, 0xffff, s2
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10001
+; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10007
+; GFX12-NEXT: s_bfe_u32 s7, s2, 0x10009
+; GFX12-NEXT: s_bfe_u32 s8, s2, 0x1000d
+; GFX12-NEXT: s_and_b32 s9, s2, 1
+; GFX12-NEXT: s_bfe_u32 s10, s2, 0x1000a
+; GFX12-NEXT: s_bfe_u32 s2, s2, 0x1000c
+; GFX12-NEXT: s_bfe_u32 s11, s6, 0x10005
+; GFX12-NEXT: s_bfe_u32 s12, s6, 0x1000b
+; GFX12-NEXT: s_lshr_b32 s13, s6, 15
+; GFX12-NEXT: s_bfe_u32 s14, s6, 0x10002
+; GFX12-NEXT: s_bfe_u32 s15, s6, 0x10006
+; GFX12-NEXT: s_bfe_u32 s16, s6, 0x10004
+; GFX12-NEXT: s_bfe_u32 s17, s6, 0x10008
+; GFX12-NEXT: s_bfe_u32 s6, s6, 0x1000e
+; GFX12-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v1, s8
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s7
+; GFX12-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v2, s6
+; GFX12-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v4, s17
+; GFX12-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s5
+; GFX12-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v8, s16
+; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s15
+; GFX12-NEXT: v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v13, s4
+; GFX12-NEXT: v_mov_b32_e32 v14, s14
+; GFX12-NEXT: s_clause 0x3
+; GFX12-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
+; GFX12-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
+; GFX12-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
+; GFX12-NEXT: global_store_b128 v16, v[12:15], s[0:1]
+; GFX12-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_zextload_v16i1_to_v16i32:
; GFX1250: ; %bb.0:
@@ -2306,11 +2280,9 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v16, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v0, v16, s[2:3] nv
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s6, 0xffff, s2
; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10001
@@ -2328,14 +2300,15 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_u32 s16, s6, 0x10004
; GFX1250-NEXT: s_bfe_u32 s17, s6, 0x10008
; GFX1250-NEXT: s_bfe_u32 s6, s6, 0x1000e
-; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
-; GFX1250-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s13
-; GFX1250-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s10
-; GFX1250-NEXT: v_dual_mov_b32 v4, s17 :: v_dual_mov_b32 v7, s12
-; GFX1250-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v12, s9
-; GFX1250-NEXT: v_dual_mov_b32 v8, s16 :: v_dual_mov_b32 v9, s11
-; GFX1250-NEXT: v_dual_mov_b32 v10, s15 :: v_dual_mov_b32 v14, s14
-; GFX1250-NEXT: v_dual_mov_b32 v13, s4 :: v_dual_mov_b32 v15, s3
+; GFX1250-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v5, s7
+; GFX1250-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v2, s6
+; GFX1250-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v4, s17
+; GFX1250-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s5
+; GFX1250-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v8, s16
+; GFX1250-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s15
+; GFX1250-NEXT: v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v13, s4
+; GFX1250-NEXT: v_mov_b32_e32 v14, s14
; GFX1250-NEXT: s_clause 0x3
; GFX1250-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX1250-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
@@ -7309,235 +7282,120 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; EG-NEXT: ADD_INT * T22.X, T15.X, literal.x,
; EG-NEXT: 28(3.923636e-44), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_zextload_v16i1_to_v16i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v3
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v3, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x1000a
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v6, 11, 1
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10009
-; GFX12-TRUE16-NEXT: v_bfe_u32 v0, v6, 8, 1
-; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x1000c
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 15, v6
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x1000d
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10007
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[0:3], s[0:1] offset:64
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10006
-; GFX12-TRUE16-NEXT: v_bfe_u32 v0, v6, 5, 1
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v6, 14, 1
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10004
-; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v0
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10001
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, 1
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[6:9], s[0:1] offset:112
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_v16i1_to_v16i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v3
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v3
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v3
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v9, v3
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v3, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x1000a
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v6, 11, 1
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10009
-; GFX12-FAKE16-NEXT: v_bfe_u32 v0, v6, 8, 1
-; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x1000c
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 15, v6
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x1000d
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10007
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[0:3], s[0:1] offset:64
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x10006
-; GFX12-FAKE16-NEXT: v_bfe_u32 v0, v6, 5, 1
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v6, 14, 1
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10004
-; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v0
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10001
-; GFX12-FAKE16-NEXT: s_and_b32 s2, s2, 1
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[6:9], s[0:1] offset:112
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-TRUE16-LABEL: constant_zextload_v16i1_to_v16i64:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-TRUE16-NEXT: v_nop
-; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, v3
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: global_load_u16 v0, v3, s[2:3] nv
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x1000a
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s3, 0x1000b
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v2, s4
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, s5
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10009
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s3, 0x10008
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s5
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s3, 0x1000e
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, s4
-; GFX1250-TRUE16-NEXT: s_lshr_b32 s4, s3, 15
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s3, s3, 0x10005
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:64
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s5
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s2, 0x1000c
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, s4
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x1000d
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:112
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s5
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s2, 0x10006
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, s4
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10007
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10004
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s2, s2, 0x10001
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s2
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[0:3], s[0:1]
-; GFX1250-TRUE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_v16i1_to_v16i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000a
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s4
+; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10009
+; GFX12-NEXT: s_bfe_u32 s5, s3, 0x1000b
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, s5
+; GFX12-NEXT: s_bfe_u32 s5, s3, 0x10008
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s5
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_lshr_b32 s4, s3, 15
+; GFX12-NEXT: s_bfe_u32 s5, s3, 0x1000e
+; GFX12-NEXT: s_bfe_u32 s3, s3, 0x10005
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s5
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000d
+; GFX12-NEXT: s_bfe_u32 s5, s2, 0x1000c
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:112
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s5
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10007
+; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10006
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s5
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10004
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10002
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:32
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10001
+; GFX12-NEXT: s_and_b32 s2, s2, 1
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:16
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1]
+; GFX12-NEXT: s_endpgm
;
-; GFX1250-FAKE16-LABEL: constant_zextload_v16i1_to_v16i64:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-FAKE16-NEXT: v_nop
-; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v5, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v9, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v13, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v15, v1 :: v_dual_mov_b32 v17, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v21, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v23, v1 :: v_dual_mov_b32 v25, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v27, v1 :: v_dual_mov_b32 v29, v1
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: global_load_u16 v12, v1, s[2:3] nv
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: s_mov_b32 s2, 0xffff
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v31, v1 :: v_dual_bitop2_b32 v22, s2, v12 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v0, v12, 10, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v6, v12, 9, 1
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v28, 1, v12
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v14, v12, 13, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v2, v22, 11, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v4, v22, 8, 1
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 15, v22
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v8, v22, 14, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v18, v12, 7, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v26, v12, 3, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v30, v12, 1, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v24, v12, 2, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v20, v12, 4, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v16, v12, 6, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v12, v12, 12, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v22, v22, 5, 1
-; GFX1250-FAKE16-NEXT: s_clause 0x7
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[4:7], s[0:1] offset:64
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[8:11], s[0:1] offset:112
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[12:15], s[0:1] offset:96
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[16:19], s[0:1] offset:48
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[20:23], s[0:1] offset:32
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[24:27], s[0:1] offset:16
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[28:31], s[0:1]
-; GFX1250-FAKE16-NEXT: s_endpgm
+; GFX1250-LABEL: constant_zextload_v16i1_to_v16i64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000a
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s4
+; GFX1250-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10009
+; GFX1250-NEXT: s_bfe_u32 s5, s3, 0x1000b
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, s5
+; GFX1250-NEXT: s_bfe_u32 s5, s3, 0x10008
+; GFX1250-NEXT: v_mov_b32_e32 v1, v3
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
+; GFX1250-NEXT: s_lshr_b32 s4, s3, 15
+; GFX1250-NEXT: s_bfe_u32 s5, s3, 0x1000e
+; GFX1250-NEXT: s_bfe_u32 s3, s3, 0x10005
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:64
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000d
+; GFX1250-NEXT: s_bfe_u32 s5, s2, 0x1000c
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:112
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10007
+; GFX1250-NEXT: s_bfe_u32 s5, s2, 0x10006
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10004
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10002
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: s_bfe_u32 s2, s2, 0x10001
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
+; GFX1250-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-NEXT: global_store_b128 v3, v[0:3], s[0:1]
+; GFX1250-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
%ext = zext <16 x i1> %load to <16 x i64>
store <16 x i64> %ext, ptr addrspace(1) %out
@@ -7820,149 +7678,77 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; EG-NEXT: MOV * T14.W, T14.Z,
; EG-NEXT: 28(3.923636e-44), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_sextload_v16i1_to_v16i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v32, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v32, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v28, s3
-; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s3, 14
-; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s3, 15
-; GFX12-TRUE16-NEXT: s_lshr_b32 s8, s3, 13
-; GFX12-TRUE16-NEXT: s_lshr_b32 s10, s3, 10
-; GFX12-TRUE16-NEXT: s_lshr_b32 s12, s3, 11
-; GFX12-TRUE16-NEXT: s_lshr_b32 s6, s3, 12
-; GFX12-TRUE16-NEXT: s_lshr_b32 s14, s3, 8
-; GFX12-TRUE16-NEXT: s_lshr_b32 s16, s3, 9
-; GFX12-TRUE16-NEXT: s_lshr_b32 s18, s3, 6
-; GFX12-TRUE16-NEXT: s_lshr_b32 s20, s3, 7
-; GFX12-TRUE16-NEXT: s_lshr_b32 s22, s3, 4
-; GFX12-TRUE16-NEXT: s_lshr_b32 s24, s3, 5
-; GFX12-TRUE16-NEXT: s_lshr_b32 s26, s3, 2
-; GFX12-TRUE16-NEXT: s_lshr_b32 s28, s3, 3
-; GFX12-TRUE16-NEXT: s_lshr_b32 s30, s3, 1
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s5
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v8, s10
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v10, s12
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v13, s15
-; GFX12-TRUE16-NEXT: v_bfe_i32 v28, v28, 0, 1
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s9
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v15, s17
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v14, s16 :: v_dual_mov_b32 v17, s19
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v19, s21
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v18, s20 :: v_dual_mov_b32 v21, s23
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v20, s22 :: v_dual_mov_b32 v23, s25
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v22, s24 :: v_dual_mov_b32 v25, s27
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v24, s26 :: v_dual_mov_b32 v27, s29
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v26, s28 :: v_dual_mov_b32 v31, s31
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v30, s30
-; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v29, 31, v28
-; GFX12-TRUE16-NEXT: s_clause 0x5
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[28:31], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_sextload_v16i1_to_v16i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v32, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v32, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v28, s3
-; GFX12-FAKE16-NEXT: s_lshr_b32 s2, s3, 14
-; GFX12-FAKE16-NEXT: s_lshr_b32 s4, s3, 15
-; GFX12-FAKE16-NEXT: s_lshr_b32 s8, s3, 13
-; GFX12-FAKE16-NEXT: s_lshr_b32 s10, s3, 10
-; GFX12-FAKE16-NEXT: s_lshr_b32 s12, s3, 11
-; GFX12-FAKE16-NEXT: s_lshr_b32 s6, s3, 12
-; GFX12-FAKE16-NEXT: s_lshr_b32 s14, s3, 8
-; GFX12-FAKE16-NEXT: s_lshr_b32 s16, s3, 9
-; GFX12-FAKE16-NEXT: s_lshr_b32 s18, s3, 6
-; GFX12-FAKE16-NEXT: s_lshr_b32 s20, s3, 7
-; GFX12-FAKE16-NEXT: s_lshr_b32 s22, s3, 4
-; GFX12-FAKE16-NEXT: s_lshr_b32 s24, s3, 5
-; GFX12-FAKE16-NEXT: s_lshr_b32 s26, s3, 2
-; GFX12-FAKE16-NEXT: s_lshr_b32 s28, s3, 3
-; GFX12-FAKE16-NEXT: s_lshr_b32 s30, s3, 1
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s5
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v8, s10
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v10, s12
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v13, s15
-; GFX12-FAKE16-NEXT: v_bfe_i32 v28, v28, 0, 1
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s9
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v15, s17
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v14, s16 :: v_dual_mov_b32 v17, s19
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v19, s21
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v18, s20 :: v_dual_mov_b32 v21, s23
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v20, s22 :: v_dual_mov_b32 v23, s25
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v22, s24 :: v_dual_mov_b32 v25, s27
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v24, s26 :: v_dual_mov_b32 v27, s29
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v26, s28 :: v_dual_mov_b32 v31, s31
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v30, s30
-; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
-; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v29, 31, v28
-; GFX12-FAKE16-NEXT: s_clause 0x5
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[28:31], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_sextload_v16i1_to_v16i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s3, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 s2, s3, 14
+; GFX12-NEXT: s_lshr_b32 s4, s3, 15
+; GFX12-NEXT: s_lshr_b32 s6, s3, 12
+; GFX12-NEXT: s_lshr_b32 s8, s3, 13
+; GFX12-NEXT: s_lshr_b32 s10, s3, 10
+; GFX12-NEXT: s_lshr_b32 s12, s3, 11
+; GFX12-NEXT: s_lshr_b32 s14, s3, 8
+; GFX12-NEXT: s_lshr_b32 s16, s3, 9
+; GFX12-NEXT: s_lshr_b32 s18, s3, 6
+; GFX12-NEXT: s_lshr_b32 s20, s3, 7
+; GFX12-NEXT: s_lshr_b32 s22, s3, 4
+; GFX12-NEXT: s_lshr_b32 s24, s3, 5
+; GFX12-NEXT: s_lshr_b32 s26, s3, 2
+; GFX12-NEXT: s_lshr_b32 s28, s3, 3
+; GFX12-NEXT: s_lshr_b32 s30, s3, 1
+; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v3, s5
+; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s7
+; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v7, s9
+; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v9, s11
+; GFX12-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v11, s13
+; GFX12-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v8, s10 :: v_dual_mov_b32 v17, s15
+; GFX12-NEXT: v_dual_mov_b32 v10, s12 :: v_dual_mov_b32 v19, s17
+; GFX12-NEXT: v_bfe_i32 v12, v12, 0, 1
+; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
+; GFX12-NEXT: v_mov_b32_e32 v16, s14
+; GFX12-NEXT: v_mov_b32_e32 v18, s16
+; GFX12-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:112
+; GFX12-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:96
+; GFX12-NEXT: v_dual_mov_b32 v0, s18 :: v_dual_mov_b32 v5, s23
+; GFX12-NEXT: v_dual_mov_b32 v1, s19 :: v_dual_mov_b32 v2, s20
+; GFX12-NEXT: v_mov_b32_e32 v7, s25
+; GFX12-NEXT: v_mov_b32_e32 v3, s21
+; GFX12-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v4, s22 :: v_dual_mov_b32 v21, s27
+; GFX12-NEXT: v_dual_mov_b32 v6, s24 :: v_dual_mov_b32 v23, s29
+; GFX12-NEXT: v_dual_mov_b32 v20, s26 :: v_dual_mov_b32 v15, s31
+; GFX12-NEXT: v_mov_b32_e32 v22, s28
+; GFX12-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GFX12-NEXT: v_mov_b32_e32 v14, s30
+; GFX12-NEXT: s_clause 0x5
+; GFX12-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:80
+; GFX12-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:64
+; GFX12-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:48
+; GFX12-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:32
+; GFX12-NEXT: global_store_b128 v24, v[20:23], s[0:1] offset:16
+; GFX12-NEXT: global_store_b128 v24, v[12:15], s[0:1]
+; GFX12-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_sextload_v16i1_to_v16i64:
; GFX1250: ; %bb.0:
@@ -7971,19 +7757,16 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v32, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v0, v32, s[2:3] nv
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_mov_b32_e32 v28, s3
+; GFX1250-NEXT: s_load_u16 s3, s[2:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v12, s3
; GFX1250-NEXT: s_lshr_b32 s2, s3, 14
; GFX1250-NEXT: s_lshr_b32 s4, s3, 15
+; GFX1250-NEXT: s_lshr_b32 s6, s3, 12
; GFX1250-NEXT: s_lshr_b32 s8, s3, 13
; GFX1250-NEXT: s_lshr_b32 s10, s3, 10
; GFX1250-NEXT: s_lshr_b32 s12, s3, 11
-; GFX1250-NEXT: s_lshr_b32 s6, s3, 12
; GFX1250-NEXT: s_lshr_b32 s14, s3, 8
; GFX1250-NEXT: s_lshr_b32 s16, s3, 9
; GFX1250-NEXT: s_lshr_b32 s18, s3, 6
@@ -7993,47 +7776,51 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-NEXT: s_lshr_b32 s26, s3, 2
; GFX1250-NEXT: s_lshr_b32 s28, s3, 3
; GFX1250-NEXT: s_lshr_b32 s30, s3, 1
-; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX1250-NEXT: v_bfe_i32 v12, v12, 0, 1
+; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
+; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
+; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v5, s7
+; GFX1250-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v7, s9
; GFX1250-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
-; GFX1250-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s9
; GFX1250-NEXT: v_dual_mov_b32 v8, s10 :: v_dual_mov_b32 v9, s11
; GFX1250-NEXT: v_dual_mov_b32 v10, s12 :: v_dual_mov_b32 v11, s13
-; GFX1250-NEXT: v_mov_b32_e32 v12, s14
-; GFX1250-NEXT: v_bfe_i32 v28, v28, 0, 1
; GFX1250-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
-; GFX1250-NEXT: v_dual_mov_b32 v13, s15 :: v_dual_mov_b32 v14, s16
-; GFX1250-NEXT: v_dual_mov_b32 v15, s17 :: v_dual_mov_b32 v16, s18
+; GFX1250-NEXT: v_dual_mov_b32 v16, s14 :: v_dual_mov_b32 v17, s15
+; GFX1250-NEXT: v_dual_mov_b32 v18, s16 :: v_dual_ashrrev_i32 v13, 31, v12
+; GFX1250-NEXT: v_mov_b32_e32 v19, s17
; GFX1250-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v17, s19 :: v_dual_mov_b32 v18, s20
-; GFX1250-NEXT: v_dual_mov_b32 v19, s21 :: v_dual_mov_b32 v20, s22
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:112
+; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:96
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_dual_mov_b32 v0, s18 :: v_dual_mov_b32 v1, s19
+; GFX1250-NEXT: v_dual_mov_b32 v2, s20 :: v_dual_mov_b32 v3, s21
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v4, s22
; GFX1250-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v21, s23 :: v_dual_mov_b32 v22, s24
-; GFX1250-NEXT: v_dual_mov_b32 v23, s25 :: v_dual_mov_b32 v24, s26
-; GFX1250-NEXT: v_dual_mov_b32 v25, s27 :: v_dual_mov_b32 v26, s28
-; GFX1250-NEXT: v_dual_mov_b32 v27, s29 :: v_dual_mov_b32 v30, s30
-; GFX1250-NEXT: v_dual_mov_b32 v31, s31 :: v_dual_ashrrev_i32 v29, 31, v28
-; GFX1250-NEXT: s_clause 0x7
-; GFX1250-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
-; GFX1250-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
-; GFX1250-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
-; GFX1250-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
-; GFX1250-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
-; GFX1250-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
-; GFX1250-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
-; GFX1250-NEXT: global_store_b128 v32, v[28:31], s[0:1]
+; GFX1250-NEXT: v_dual_mov_b32 v5, s23 :: v_dual_mov_b32 v6, s24
+; GFX1250-NEXT: v_dual_mov_b32 v7, s25 :: v_dual_mov_b32 v20, s26
+; GFX1250-NEXT: v_dual_mov_b32 v21, s27 :: v_dual_mov_b32 v22, s28
+; GFX1250-NEXT: v_dual_mov_b32 v23, s29 :: v_dual_mov_b32 v14, s30
+; GFX1250-NEXT: v_mov_b32_e32 v15, s31
+; GFX1250-NEXT: s_clause 0x5
+; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:80
+; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:64
+; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:48
+; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:32
+; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[0:1] offset:16
+; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
%ext = sext <16 x i1> %load to <16 x i64>
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
index 1536ac5112e7b1..b06fff0320b3ef 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -84,18 +84,19 @@ define amdgpu_kernel void @constant_load_i16(ptr addrspace(1) %out, ptr addrspac
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX12-TRUE16-NEXT: s_endpgm
;
; GFX12-FAKE16-LABEL: constant_load_i16:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX12-FAKE16-NEXT: s_endpgm
entry:
@@ -765,29 +766,31 @@ define amdgpu_kernel void @constant_load_v16i16_align2(ptr addrspace(4) %ptr0) #
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_clause 0x7
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v3, v8, s[0:1] offset:12
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v7, v8, s[0:1] offset:28
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v6, v8, s[0:1] offset:24
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v5, v8, s[0:1] offset:20
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v4, v8, s[0:1] offset:16
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v2, v8, s[0:1] offset:8
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v1, v8, s[0:1] offset:4
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v8, s[0:1]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-TRUE16-NEXT: s_load_u16 s2, s[0:1], 0xc
+; GFX12-TRUE16-NEXT: s_load_u16 s3, s[0:1], 0x8
+; GFX12-TRUE16-NEXT: s_load_u16 s4, s[0:1], 0x4
+; GFX12-TRUE16-NEXT: s_load_u16 s5, s[0:1], 0x0
+; GFX12-TRUE16-NEXT: s_load_u16 s6, s[0:1], 0x1c
+; GFX12-TRUE16-NEXT: s_load_u16 s7, s[0:1], 0x18
+; GFX12-TRUE16-NEXT: s_load_u16 s8, s[0:1], 0x14
+; GFX12-TRUE16-NEXT: s_load_u16 s9, s[0:1], 0x10
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, s3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, s4
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, s6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, s7
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, s8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, s9
+; GFX12-TRUE16-NEXT: s_clause 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v7, v8, s[0:1] offset:30
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v6, v8, s[0:1] offset:26
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v5, v8, s[0:1] offset:22
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v4, v8, s[0:1] offset:18
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v1, v8, s[0:1] offset:6
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v0, v8, s[0:1] offset:2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_clause 0x1
@@ -798,32 +801,30 @@ define amdgpu_kernel void @constant_load_v16i16_align2(ptr addrspace(4) %ptr0) #
; GFX12-FAKE16-LABEL: constant_load_v16i16_align2:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, 0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: s_clause 0x7
-; GFX12-FAKE16-NEXT: global_load_u16 v3, v8, s[0:1] offset:12
-; GFX12-FAKE16-NEXT: global_load_u16 v7, v8, s[0:1] offset:28
-; GFX12-FAKE16-NEXT: global_load_u16 v6, v8, s[0:1] offset:24
-; GFX12-FAKE16-NEXT: global_load_u16 v5, v8, s[0:1] offset:20
-; GFX12-FAKE16-NEXT: global_load_u16 v4, v8, s[0:1] offset:16
-; GFX12-FAKE16-NEXT: global_load_u16 v2, v8, s[0:1] offset:8
-; GFX12-FAKE16-NEXT: global_load_u16 v1, v8, s[0:1] offset:4
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v8, s[0:1]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-FAKE16-NEXT: s_load_u16 s2, s[0:1], 0xc
+; GFX12-FAKE16-NEXT: s_load_u16 s3, s[0:1], 0x8
+; GFX12-FAKE16-NEXT: s_load_u16 s4, s[0:1], 0x4
+; GFX12-FAKE16-NEXT: s_load_u16 s5, s[0:1], 0x0
+; GFX12-FAKE16-NEXT: s_load_u16 s6, s[0:1], 0x1c
+; GFX12-FAKE16-NEXT: s_load_u16 s7, s[0:1], 0x18
+; GFX12-FAKE16-NEXT: s_load_u16 s8, s[0:1], 0x14
+; GFX12-FAKE16-NEXT: s_load_u16 s9, s[0:1], 0x10
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v3, s2
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v7, s6
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, s7 :: v_dual_mov_b32 v5, s8
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s9
+; GFX12-FAKE16-NEXT: s_clause 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v7, v8, s[0:1] offset:30
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v6, v8, s[0:1] offset:26
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v5, v8, s[0:1] offset:22
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v4, v8, s[0:1] offset:18
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v1, v8, s[0:1] offset:6
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v0, v8, s[0:1] offset:2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: s_clause 0x1
@@ -5423,27 +5424,17 @@ define amdgpu_kernel void @constant_zextload_i16_to_i64(ptr addrspace(1) %out, p
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_zextload_i16_to_i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-TRUE16-NEXT: global_store_b64 v1, v[0:1], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_i16_to_i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v1, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-FAKE16-NEXT: global_store_b64 v1, v[0:1], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_i16_to_i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-NEXT: global_store_b64 v1, v[0:1], s[0:1]
+; GFX12-NEXT: s_endpgm
%a = load i16, ptr addrspace(4) %in
%ext = zext i16 %a to i64
store i64 %ext, ptr addrspace(1) %out
@@ -5525,31 +5516,19 @@ define amdgpu_kernel void @constant_sextload_i16_to_i64(ptr addrspace(1) %out, p
; EG-NEXT: ASHR * T0.Y, PV.X, literal.x,
; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_sextload_i16_to_i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v2, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_sextload_i16_to_i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v2, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_sextload_i16_to_i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_sext_i32_i16 s2, s2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_ashr_i32 s3, s2, 31
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT: s_endpgm
%a = load i16, ptr addrspace(4) %in
%ext = sext i16 %a to i64
store i64 %ext, ptr addrspace(1) %out
@@ -5624,27 +5603,17 @@ define amdgpu_kernel void @constant_zextload_v1i16_to_v1i64(ptr addrspace(1) %ou
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_zextload_v1i16_to_v1i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-TRUE16-NEXT: global_store_b64 v1, v[0:1], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_v1i16_to_v1i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v1, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-FAKE16-NEXT: global_store_b64 v1, v[0:1], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_v1i16_to_v1i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-NEXT: global_store_b64 v1, v[0:1], s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <1 x i16>, ptr addrspace(4) %in
%ext = zext <1 x i16> %load to <1 x i64>
store <1 x i64> %ext, ptr addrspace(1) %out
@@ -5721,31 +5690,19 @@ define amdgpu_kernel void @constant_sextload_v1i16_to_v1i64(ptr addrspace(1) %ou
; EG-NEXT: ASHR * T0.Y, PV.X, literal.x,
; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_sextload_v1i16_to_v1i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v2, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_sextload_v1i16_to_v1i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v2, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_sextload_v1i16_to_v1i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_sext_i32_i16 s2, s2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_ashr_i32 s3, s2, 31
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <1 x i16>, ptr addrspace(4) %in
%ext = sext <1 x i16> %load to <1 x i64>
store <1 x i64> %ext, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
index dccc12ae2a03c0..20d5d812779b8d 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
@@ -183,18 +183,19 @@ define amdgpu_kernel void @constant_load_v2i8(ptr addrspace(1) %out, ptr addrspa
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX12-TRUE16-NEXT: s_endpgm
;
; GFX12-FAKE16-LABEL: constant_load_v2i8:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX12-FAKE16-NEXT: s_endpgm
entry:
@@ -944,33 +945,20 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i32(ptr addrspace(1) %out
; EG-NEXT: LSHR * T5.X, KC0[2].Y, literal.y,
; EG-NEXT: 255(3.573311e-43), 2(2.802597e-45)
;
-; GFX12-TRUE16-LABEL: constant_zextload_v2i8_to_v2i32:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v2, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_v2i8_to_v2i32:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v2, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_v2i8_to_v2i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX12-NEXT: s_and_b32 s2, s2, 0xff
+; GFX12-NEXT: s_lshr_b32 s3, s3, 8
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
%ext = zext <2 x i8> %load to <2 x i32>
store <2 x i32> %ext, ptr addrspace(1) %out
@@ -5632,37 +5620,21 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i64(ptr addrspace(1) %out
; EG-NEXT: LSHR * T5.X, KC0[2].Y, literal.y,
; EG-NEXT: 255(3.573311e-43), 2(2.802597e-45)
;
-; GFX12-TRUE16-LABEL: constant_zextload_v2i8_to_v2i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
-; GFX12-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_v2i8_to_v2i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v1, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
-; GFX12-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_v2i8_to_v2i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX12-NEXT: s_and_b32 s2, s2, 0xff
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-NEXT: s_lshr_b32 s3, s3, 8
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, v1
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
%ext = zext <2 x i8> %load to <2 x i64>
store <2 x i64> %ext, ptr addrspace(1) %out
@@ -5762,39 +5734,25 @@ define amdgpu_kernel void @constant_sextload_v2i8_to_v2i64(ptr addrspace(1) %out
; EG-NEXT: ASHR * T4.W, PV.Z, literal.y,
; EG-NEXT: 2(2.802597e-45), 31(4.344025e-44)
;
-; GFX12-TRUE16-LABEL: constant_sextload_v2i8_to_v2i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v4, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v1, 0, 8
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX12-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_sextload_v2i8_to_v2i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v4, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX12-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_i32 v2, v1, 0, 8
-; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX12-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_sextload_v2i8_to_v2i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: s_lshr_b32 s2, s2, 8
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x80000
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v3, s3
+; GFX12-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX12-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
%ext = sext <2 x i8> %load to <2 x i64>
store <2 x i64> %ext, ptr addrspace(1) %out
@@ -9502,35 +9460,20 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i16(ptr addrspace(1) %out
; EG-NEXT: LSHR * T6.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_zextload_v2i8_to_v2i16:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v1, v0, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_v2i8_to_v2i16:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v1
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
-; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX12-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_v2i8_to_v2i16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX12-NEXT: s_and_b32 s2, s2, 0xff
+; GFX12-NEXT: s_lshr_b32 s3, s3, 8
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
%ext = zext <2 x i8> %load to <2 x i16>
store <2 x i16> %ext, ptr addrspace(1) %out
@@ -9632,37 +9575,20 @@ define amdgpu_kernel void @constant_sextload_v2i8_to_v2i16(ptr addrspace(1) %out
; EG-NEXT: LSHR * T6.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_sextload_v2i8_to_v2i16:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v1, v0, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v1, 0, 16
-; GFX12-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_sextload_v2i8_to_v2i16:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_bfe_i32 v2, v1, 0, 16
-; GFX12-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX12-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_sextload_v2i8_to_v2i16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_sext_i32_i16 s3, s2
+; GFX12-NEXT: s_bfe_i32 s2, s2, 0x80000
+; GFX12-NEXT: s_lshr_b32 s3, s3, 8
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
%ext = sext <2 x i8> %load to <2 x i16>
store <2 x i16> %ext, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index cab1b8a45ec5a9..6c1c11aed600bf 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -3324,17 +3324,17 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_load_u16 v1, v0, s[14:15]
-; GFX1250-NEXT: global_load_u16 v2, v0, s[12:13]
-; GFX1250-NEXT: s_wait_loadcnt 0x1
-; GFX1250-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v4, 0xffff, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, v4, v3
-; GFX1250-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: s_load_u16 s0, s[14:15], 0x0
+; GFX1250-NEXT: s_load_u16 s1, s[12:13], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_and_b32 s2, 0xffff, s0
+; GFX1250-NEXT: s_and_b32 s3, 0xffff, s1
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_lt_u32 s3, s2
+; GFX1250-NEXT: s_cselect_b32 s0, s1, s0
+; GFX1250-NEXT: s_cselect_b32 s1, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s1
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_b16 v0, v1, s[8:9]
; GFX1250-NEXT: global_store_b8 v0, v2, s[10:11]
diff --git a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
index 60078fde83b56b..d8ce9f1726965a 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
@@ -298,17 +298,30 @@ define amdgpu_kernel void @incorrect_type_i16_block_count_x(ptr addrspace(1) inr
; GFX90a-NEXT: global_store_short v0, v1, s[8:9]
; GFX90a-NEXT: s_endpgm
;
-; GFX1250-LABEL: incorrect_type_i16_block_count_x:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1] offset:8 nv
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: global_store_b16 v0, v1, s[2:3]
-; GFX1250-NEXT: s_endpgm
+; GFX1250-FAKE16-LABEL: incorrect_type_i16_block_count_x:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: s_load_u16 s0, s[0:1], 0x8 nv
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-FAKE16-NEXT: global_store_b16 v0, v1, s[2:3]
+; GFX1250-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-REAL16-LABEL: incorrect_type_i16_block_count_x:
+; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-REAL16-NEXT: v_nop
+; GFX1250-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-REAL16-NEXT: s_load_u16 s0, s[0:1], 0x8 nv
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1250-REAL16-NEXT: global_store_b16 v1, v0, s[2:3]
+; GFX1250-REAL16-NEXT: s_endpgm
%imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
%load = load i16, ptr addrspace(4) %imp_arg_ptr
store i16 %load, ptr addrspace(1) %out
@@ -1172,6 +1185,3 @@ define amdgpu_kernel void @preload_block_count_z_workgroup_size_z_remainder_z(pt
}
attributes #0 = { "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX1250-FAKE16: {{.*}}
-; GFX1250-REAL16: {{.*}}
>From aa6c2cc6b221bfc02a97e71406d4c7b0fd5c9714 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Wed, 23 Sep 2026 05:19:43 -0500
Subject: [PATCH 2/2] Use legalization approach
Signed-off-by: John Lu <John.Lu at amd.com>
---
llvm/include/llvm/CodeGen/TargetLowering.h | 6 +
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 2 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 41 ++
llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 +
llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll | 4 +-
llvm/test/CodeGen/AMDGPU/fcmp.f16.ll | 88 ++--
llvm/test/CodeGen/AMDGPU/fmax3.ll | 48 +-
llvm/test/CodeGen/AMDGPU/fmaximum.ll | 6 +-
llvm/test/CodeGen/AMDGPU/fmin3.ll | 48 +-
llvm/test/CodeGen/AMDGPU/fminimum.ll | 6 +-
llvm/test/CodeGen/AMDGPU/frem.ll | 351 ++++++-------
llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 461 ++++++++----------
llvm/test/CodeGen/AMDGPU/load-constant-i16.ll | 158 +++---
llvm/test/CodeGen/AMDGPU/load-constant-i8.ll | 65 +--
llvm/test/CodeGen/AMDGPU/min.ll | 11 +-
...-to-valu-pseudo-scalar-trans-f16-true16.ll | 50 +-
.../AMDGPU/preload-implicit-kernargs.ll | 39 +-
23 files changed, 662 insertions(+), 735 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index cd6ec343a8bbab..54c5a153f20db7 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4811,6 +4811,12 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
return isTypeLegal(VT);
}
+ /// Overload that provides context about the specific node being optimized.
+ /// By default, calls the version without the node parameter.
+ virtual bool isTypeDesirableForOp(unsigned Opc, EVT VT, SDNode *N) const {
+ return isTypeDesirableForOp(Opc, VT);
+ }
+
/// Return true if it is profitable for dag combiner to transform a floating
/// point op of specified opcode to a equivalent op of an integer
/// type. e.g. f32 load -> i32 load can be profitable on ARM.
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index d4a7403a6b25fa..941728a7461a1a 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -17884,7 +17884,7 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
// fold (truncate (load x)) -> (smaller load x)
// fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
- if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) {
+ if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT, N0.getNode())) {
if (SDValue Reduced = reduceLoadWidth(N))
return Reduced;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index b5fa732ea9418c..f819c149011db0 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -2505,6 +2505,24 @@ bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
return TargetLowering::isTypeDesirableForOp(Op, VT);
}
+bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT,
+ SDNode *N) const {
+ // Do not convert uniform i32 loads to 16-bit.
+ // Uniform 16-bit loads are legalized to i16 = trunc (zextload i16->i32)
+ // to match subword load patterns. Allowing conversion back to a 16-bit
+ // load would create an infinite loop.
+ if (Subtarget->hasScalarSubwordLoads() && Op == ISD::LOAD &&
+ !VT.isVector() && VT.getSizeInBits() == 16) {
+ auto *Load = dyn_cast<LoadSDNode>(N);
+ if (Load && Load->getValueType(0) == MVT::i32 && !Load->isDivergent() &&
+ AMDGPU::isUniformMMO(Load->getMemOperand())) {
+ return false;
+ }
+ }
+
+ return isTypeDesirableForOp(Op, VT);
+}
+
MachinePointerInfo
SITargetLowering::getKernargSegmentPtrInfo(MachineFunction &MF) const {
// This isn't really a constant pool but close enough.
@@ -13562,6 +13580,29 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
MachineMemOperand *MMO = Load->getMemOperand();
if (ExtType == ISD::NON_EXTLOAD && MemVT.getSizeInBits() < 32) {
+ // Legalize uniform 16-bit loads to i16 = trunc (zextload i16->i32)
+ // to match subword load patterns.
+ if (!MemVT.isVector() && MemVT.getSizeInBits() == 16 &&
+ isTypeLegal(MemVT) && Subtarget->hasScalarSubwordLoads() &&
+ !Load->isDivergent() && AMDGPU::isUniformMMO(MMO)) {
+ SDValue Chain = Load->getChain();
+ SDValue BasePtr = Load->getBasePtr();
+
+ // Load as i16 and zero-extend to i32 (matches S_LOAD_U16 behavior)
+ SDValue NewLD = DAG.getExtLoad(ISD::ZEXTLOAD, DL, MVT::i32, Chain,
+ BasePtr, MVT::i16, MMO);
+
+ // Truncate back to i16
+ SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, NewLD);
+
+ // For f16/bf16, bitcast from i16 to the original fp type
+ SDValue Result = (MemVT == MVT::i16) ? Trunc :
+ DAG.getNode(ISD::BITCAST, DL, MemVT, Trunc);
+
+ SDValue Ops[] = {Result, NewLD.getValue(1)};
+ return DAG.getMergeValues(Ops, DL);
+ }
+
if (MemVT == MVT::i16 && isTypeLegal(MVT::i16))
return SDValue();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index fc52aa4eeb5a49..252c64ee0b4bc2 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -416,6 +416,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override;
bool isTypeDesirableForOp(unsigned Op, EVT VT) const override;
+ bool isTypeDesirableForOp(unsigned Op, EVT VT, SDNode *N) const override;
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
diff --git a/llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll b/llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll
index 7fe521102be992..25a488f2b795fb 100644
--- a/llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll
+++ b/llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll
@@ -126,9 +126,9 @@ define amdgpu_kernel void @d16_two_order_group(ptr addrspace(3) %in1, ptr %in2)
; GFX12-NEXT: flat_load_d16_b16 v0, v[1:2]
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v3, s0
-; GFX12-NEXT: ds_load_u16_d16_hi v0, v3
+; GFX12-NEXT: ds_load_u16 v3, v3
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_or_b16 v0.l, v0.h, v0.l
+; GFX12-NEXT: v_or_b16 v0.l, v3.l, v0.l
; GFX12-NEXT: flat_store_b16 v[1:2], v0
; GFX12-NEXT: s_endpgm
%i16a = load i16, ptr addrspace(3) %in1, align 2
diff --git a/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll b/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
index d2c4f31d816ecc..a2fa03c1588c6c 100644
--- a/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
@@ -131,13 +131,13 @@ define amdgpu_kernel void @fcmp_f16_lt(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -303,9 +303,9 @@ define amdgpu_kernel void @fcmp_f16_lt_abs(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
@@ -483,13 +483,13 @@ define amdgpu_kernel void @fcmp_f16_eq(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_eq_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -655,13 +655,13 @@ define amdgpu_kernel void @fcmp_f16_le(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -827,13 +827,13 @@ define amdgpu_kernel void @fcmp_f16_gt(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -999,13 +999,13 @@ define amdgpu_kernel void @fcmp_f16_lg(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1171,13 +1171,13 @@ define amdgpu_kernel void @fcmp_f16_ge(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_ge_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_ge_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1343,13 +1343,13 @@ define amdgpu_kernel void @fcmp_f16_o(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1515,13 +1515,13 @@ define amdgpu_kernel void @fcmp_f16_u(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1687,13 +1687,13 @@ define amdgpu_kernel void @fcmp_f16_nge(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1859,13 +1859,13 @@ define amdgpu_kernel void @fcmp_f16_nlg(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_nlg_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_nlg_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -2031,13 +2031,13 @@ define amdgpu_kernel void @fcmp_f16_ngt(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_ngt_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_ngt_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -2203,13 +2203,13 @@ define amdgpu_kernel void @fcmp_f16_nle(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -2375,13 +2375,13 @@ define amdgpu_kernel void @fcmp_f16_neq(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_neq_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_neq_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -2547,13 +2547,13 @@ define amdgpu_kernel void @fcmp_f16_nlt(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_nlt_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_nlt_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fmax3.ll b/llvm/test/CodeGen/AMDGPU/fmax3.ll
index f34efb4f4af2ba..6618966a8e1ce4 100644
--- a/llvm/test/CodeGen/AMDGPU/fmax3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmax3.ll
@@ -696,15 +696,15 @@ define amdgpu_kernel void @test_fmax3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
; GFX12-TRUE16-NEXT: s_mov_b32 s17, s5
; GFX12-TRUE16-NEXT: s_mov_b32 s20, s6
; GFX12-TRUE16-NEXT: s_mov_b32 s21, s7
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v2, off, s[16:19], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[20:23], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_max3_num_f16 v0.l, v0.l, v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_max3_num_f16 v0.l, v1.l, v2.l, v0.l
; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -760,15 +760,15 @@ define amdgpu_kernel void @test_fmax3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
; GFX1250-TRUE16-NEXT: s_mov_b32 s17, s13
; GFX1250-TRUE16-NEXT: s_mov_b32 s20, s14
; GFX1250-TRUE16-NEXT: s_mov_b32 s21, s15
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v2, off, s[16:19], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v0, off, s[20:23], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s0, s8
; GFX1250-TRUE16-NEXT: s_mov_b32 s1, s9
-; GFX1250-TRUE16-NEXT: v_max3_num_f16 v0.l, v0.l, v0.h, v1.l
+; GFX1250-TRUE16-NEXT: v_max3_num_f16 v0.l, v1.l, v2.l, v0.l
; GFX1250-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-TRUE16-NEXT: s_endpgm
;
@@ -1044,24 +1044,24 @@ define amdgpu_kernel void @test_fmax3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
; GFX12-TRUE16-NEXT: s_mov_b32 s15, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s18, s10
; GFX12-TRUE16-NEXT: s_mov_b32 s19, s11
-; GFX12-TRUE16-NEXT: s_mov_b32 s22, s10
-; GFX12-TRUE16-NEXT: s_mov_b32 s23, s11
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s16, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s17, s5
-; GFX12-TRUE16-NEXT: s_mov_b32 s20, s6
-; GFX12-TRUE16-NEXT: s_mov_b32 s21, s7
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s11
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v2, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_max3_num_f16 v0.l, v1.l, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_max3_num_f16 v0.l, v2.l, v1.l, v0.l
; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -1108,24 +1108,24 @@ define amdgpu_kernel void @test_fmax3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
; GFX1250-TRUE16-NEXT: s_mov_b32 s7, s3
; GFX1250-TRUE16-NEXT: s_mov_b32 s18, s2
; GFX1250-TRUE16-NEXT: s_mov_b32 s19, s3
-; GFX1250-TRUE16-NEXT: s_mov_b32 s22, s2
-; GFX1250-TRUE16-NEXT: s_mov_b32 s23, s3
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s4, s10
; GFX1250-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1250-TRUE16-NEXT: s_mov_b32 s16, s12
; GFX1250-TRUE16-NEXT: s_mov_b32 s17, s13
-; GFX1250-TRUE16-NEXT: s_mov_b32 s20, s14
-; GFX1250-TRUE16-NEXT: s_mov_b32 s21, s15
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: s_mov_b32 s12, s14
+; GFX1250-TRUE16-NEXT: s_mov_b32 s13, s15
+; GFX1250-TRUE16-NEXT: s_mov_b32 s14, s2
+; GFX1250-TRUE16-NEXT: s_mov_b32 s15, s3
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v2, off, s[12:15], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s0, s8
; GFX1250-TRUE16-NEXT: s_mov_b32 s1, s9
-; GFX1250-TRUE16-NEXT: v_max3_num_f16 v0.l, v1.l, v0.l, v0.h
+; GFX1250-TRUE16-NEXT: v_max3_num_f16 v0.l, v2.l, v1.l, v0.l
; GFX1250-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
index 213d6ffeb9f00e..dffa28a5c52fd2 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -1492,11 +1492,11 @@ define amdgpu_kernel void @fmaximum_f16_move_to_valu(ptr addrspace(1) %out, ptr
; GFX12-SDAG-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-SDAG-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3] scope:SCOPE_SYS
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: global_load_d16_hi_b16 v0, v1, s[4:5] scope:SCOPE_SYS
+; GFX12-SDAG-TRUE16-NEXT: global_load_u16 v2, v1, s[4:5] scope:SCOPE_SYS
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fmin3.ll b/llvm/test/CodeGen/AMDGPU/fmin3.ll
index c03e81e6b71299..bcd50ee8f47fd6 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin3.ll
@@ -696,15 +696,15 @@ define amdgpu_kernel void @test_fmin3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
; GFX12-TRUE16-NEXT: s_mov_b32 s17, s5
; GFX12-TRUE16-NEXT: s_mov_b32 s20, s6
; GFX12-TRUE16-NEXT: s_mov_b32 s21, s7
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v2, off, s[16:19], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[20:23], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_min3_num_f16 v0.l, v0.l, v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_min3_num_f16 v0.l, v1.l, v2.l, v0.l
; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -760,15 +760,15 @@ define amdgpu_kernel void @test_fmin3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
; GFX1250-TRUE16-NEXT: s_mov_b32 s17, s13
; GFX1250-TRUE16-NEXT: s_mov_b32 s20, s14
; GFX1250-TRUE16-NEXT: s_mov_b32 s21, s15
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v2, off, s[16:19], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v0, off, s[20:23], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s0, s8
; GFX1250-TRUE16-NEXT: s_mov_b32 s1, s9
-; GFX1250-TRUE16-NEXT: v_min3_num_f16 v0.l, v0.l, v0.h, v1.l
+; GFX1250-TRUE16-NEXT: v_min3_num_f16 v0.l, v1.l, v2.l, v0.l
; GFX1250-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-TRUE16-NEXT: s_endpgm
;
@@ -1044,24 +1044,24 @@ define amdgpu_kernel void @test_fmin3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
; GFX12-TRUE16-NEXT: s_mov_b32 s15, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s18, s10
; GFX12-TRUE16-NEXT: s_mov_b32 s19, s11
-; GFX12-TRUE16-NEXT: s_mov_b32 s22, s10
-; GFX12-TRUE16-NEXT: s_mov_b32 s23, s11
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s16, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s17, s5
-; GFX12-TRUE16-NEXT: s_mov_b32 s20, s6
-; GFX12-TRUE16-NEXT: s_mov_b32 s21, s7
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s11
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v2, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_min3_num_f16 v0.l, v1.l, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_min3_num_f16 v0.l, v2.l, v1.l, v0.l
; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -1108,24 +1108,24 @@ define amdgpu_kernel void @test_fmin3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
; GFX1250-TRUE16-NEXT: s_mov_b32 s7, s3
; GFX1250-TRUE16-NEXT: s_mov_b32 s18, s2
; GFX1250-TRUE16-NEXT: s_mov_b32 s19, s3
-; GFX1250-TRUE16-NEXT: s_mov_b32 s22, s2
-; GFX1250-TRUE16-NEXT: s_mov_b32 s23, s3
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s4, s10
; GFX1250-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1250-TRUE16-NEXT: s_mov_b32 s16, s12
; GFX1250-TRUE16-NEXT: s_mov_b32 s17, s13
-; GFX1250-TRUE16-NEXT: s_mov_b32 s20, s14
-; GFX1250-TRUE16-NEXT: s_mov_b32 s21, s15
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: s_mov_b32 s12, s14
+; GFX1250-TRUE16-NEXT: s_mov_b32 s13, s15
+; GFX1250-TRUE16-NEXT: s_mov_b32 s14, s2
+; GFX1250-TRUE16-NEXT: s_mov_b32 s15, s3
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v2, off, s[12:15], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s0, s8
; GFX1250-TRUE16-NEXT: s_mov_b32 s1, s9
-; GFX1250-TRUE16-NEXT: v_min3_num_f16 v0.l, v1.l, v0.l, v0.h
+; GFX1250-TRUE16-NEXT: v_min3_num_f16 v0.l, v2.l, v1.l, v0.l
; GFX1250-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
index 094dd76e0f945b..11acac09aa30f4 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -1492,11 +1492,11 @@ define amdgpu_kernel void @fminimum_f16_move_to_valu(ptr addrspace(1) %out, ptr
; GFX12-SDAG-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-SDAG-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3] scope:SCOPE_SYS
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: global_load_d16_hi_b16 v0, v1, s[4:5] scope:SCOPE_SYS
+; GFX12-SDAG-TRUE16-NEXT: global_load_u16 v2, v1, s[4:5] scope:SCOPE_SYS
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index 486d7c15150bc9..435aa898a6c33b 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -977,244 +977,249 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX1200-TRUE16-LABEL: frem_f16:
; GFX1200-TRUE16: ; %bb.0:
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1200-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1200-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v0, 0
; GFX1200-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[10:11]
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v1, v1, s[0:1] offset:8
+; GFX1200-TRUE16-NEXT: global_load_u16 v1, v0, s[2:3]
+; GFX1200-TRUE16-NEXT: global_load_u16 v0, v0, s[4:5] offset:8
; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x1
-; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s6, v1
; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1200-TRUE16-NEXT: s_and_b32 s0, s0, 0x7fff
-; GFX1200-TRUE16-NEXT: s_and_b32 s2, s1, 0x7fff
-; GFX1200-TRUE16-NEXT: s_cvt_f32_f16 s1, s0
-; GFX1200-TRUE16-NEXT: s_cvt_f32_f16 s0, s2
+; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1200-TRUE16-NEXT: s_and_b32 s2, s6, 0x7fff
+; GFX1200-TRUE16-NEXT: s_and_b32 s4, s3, 0x7fff
+; GFX1200-TRUE16-NEXT: s_cvt_f32_f16 s5, s2
+; GFX1200-TRUE16-NEXT: s_cvt_f32_f16 s4, s4
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1200-TRUE16-NEXT: s_cmp_ngt_f32 s1, s0
+; GFX1200-TRUE16-NEXT: s_cmp_ngt_f32 s5, s4
; GFX1200-TRUE16-NEXT: s_cbranch_scc0 .LBB0_2
; GFX1200-TRUE16-NEXT: ; %bb.1: ; %frem.else
-; GFX1200-TRUE16-NEXT: s_cmp_eq_f32 s1, s0
-; GFX1200-TRUE16-NEXT: v_and_b16 v0.h, 0x8000, v0.l
-; GFX1200-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1200-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.l, v0.h, s2
-; GFX1200-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX1200-TRUE16-NEXT: s_cmp_eq_f32 s5, s4
+; GFX1200-TRUE16-NEXT: v_and_b16 v0.l, 0x8000, s6
+; GFX1200-TRUE16-NEXT: s_cselect_b32 s7, -1, 0
+; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT: v_cndmask_b16 v0.l, s6, v0.l, s7
+; GFX1200-TRUE16-NEXT: s_mov_b32 s6, 0
; GFX1200-TRUE16-NEXT: s_branch .LBB0_3
; GFX1200-TRUE16-NEXT: .LBB0_2:
-; GFX1200-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1200-TRUE16-NEXT: ; implicit-def: $vgpr2
+; GFX1200-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1200-TRUE16-NEXT: ; implicit-def: $vgpr0
; GFX1200-TRUE16-NEXT: .LBB0_3: ; %Flow18
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1200-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1200-TRUE16-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX1200-TRUE16-NEXT: s_cselect_b32 s6, 1, 0
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX1200-TRUE16-NEXT: s_cmp_lg_u32 s6, 1
; GFX1200-TRUE16-NEXT: s_cbranch_scc1 .LBB0_8
; GFX1200-TRUE16-NEXT: ; %bb.4: ; %frem.compute
-; GFX1200-TRUE16-NEXT: v_frexp_mant_f32_e32 v3, s0
-; GFX1200-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v5, s0
-; GFX1200-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v4, s1
-; GFX1200-TRUE16-NEXT: v_frexp_mant_f32_e32 v2, s1
+; GFX1200-TRUE16-NEXT: v_frexp_mant_f32_e32 v1, s4
+; GFX1200-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v3, s4
+; GFX1200-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v2, s5
+; GFX1200-TRUE16-NEXT: v_frexp_mant_f32_e32 v0, s5
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-TRUE16-NEXT: v_ldexp_f32 v3, v3, 1
-; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s0, v5
-; GFX1200-TRUE16-NEXT: v_add_nc_u32_e32 v5, -1, v5
-; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s1, v4
-; GFX1200-TRUE16-NEXT: v_ldexp_f32 v2, v2, 11
-; GFX1200-TRUE16-NEXT: v_div_scale_f32 v6, null, v3, v3, 1.0
+; GFX1200-TRUE16-NEXT: v_ldexp_f32 v1, v1, 1
+; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s4, v3
+; GFX1200-TRUE16-NEXT: v_add_nc_u32_e32 v3, -1, v3
+; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
+; GFX1200-TRUE16-NEXT: v_ldexp_f32 v0, v0, 11
+; GFX1200-TRUE16-NEXT: v_div_scale_f32 v4, null, v1, v1, 1.0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-TRUE16-NEXT: v_not_b32_e32 v5, v5
-; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v7, v6
+; GFX1200-TRUE16-NEXT: v_not_b32_e32 v3, v3
+; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v5, v4
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(TRANS32_DEP_1)
-; GFX1200-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v4
-; GFX1200-TRUE16-NEXT: v_div_scale_f32 v4, vcc_lo, 1.0, v3, 1.0
+; GFX1200-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v2
+; GFX1200-TRUE16-NEXT: v_div_scale_f32 v2, vcc_lo, 1.0, v1, 1.0
; GFX1200-TRUE16-NEXT: s_denorm_mode 15
-; GFX1200-TRUE16-NEXT: v_fma_f32 v8, -v6, v7, 1.0
+; GFX1200-TRUE16-NEXT: v_fma_f32 v6, -v4, v5, 1.0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v7, v8, v7
-; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v8, v4, v7
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v5, v6, v5
+; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v6, v2, v5
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_f32 v9, -v6, v8, v4
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v8, v9, v7
+; GFX1200-TRUE16-NEXT: v_fma_f32 v7, -v4, v6, v2
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v6, v7, v5
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_f32 v4, -v6, v8, v4
+; GFX1200-TRUE16-NEXT: v_fma_f32 v2, -v4, v6, v2
; GFX1200-TRUE16-NEXT: s_denorm_mode 12
-; GFX1200-TRUE16-NEXT: v_div_fmas_f32 v4, v4, v7, v8
-; GFX1200-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v5
+; GFX1200-TRUE16-NEXT: v_div_fmas_f32 v2, v2, v5, v6
+; GFX1200-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v3
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-TRUE16-NEXT: v_div_fixup_f32 v4, v4, v3, 1.0
+; GFX1200-TRUE16-NEXT: v_div_fixup_f32 v2, v2, v1, 1.0
; GFX1200-TRUE16-NEXT: s_cbranch_vccnz .LBB0_7
; GFX1200-TRUE16-NEXT: ; %bb.5: ; %frem.loop_body.preheader
-; GFX1200-TRUE16-NEXT: s_sub_co_i32 s0, s1, s0
+; GFX1200-TRUE16-NEXT: s_sub_co_i32 s4, s5, s4
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_add_co_i32 s0, s0, 11
+; GFX1200-TRUE16-NEXT: s_add_co_i32 s4, s4, 11
; GFX1200-TRUE16-NEXT: .LBB0_6: ; %frem.loop_body
; GFX1200-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v5, v2, v4
+; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v3, v0, v2
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_add_co_i32 s0, s0, -11
+; GFX1200-TRUE16-NEXT: s_add_co_i32 s4, s4, -11
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_cmp_gt_i32 s0, 11
-; GFX1200-TRUE16-NEXT: v_rndne_f32_e32 v5, v5
+; GFX1200-TRUE16-NEXT: s_cmp_gt_i32 s4, 11
+; GFX1200-TRUE16-NEXT: v_rndne_f32_e32 v3, v3
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v2, v5, v3
+; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v0, v3, v1
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v2
-; GFX1200-TRUE16-NEXT: v_add_f32_e32 v5, v2, v3
+; GFX1200-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v0
+; GFX1200-TRUE16-NEXT: v_add_f32_e32 v3, v0, v1
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
+; GFX1200-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_ldexp_f32 v2, v2, 11
+; GFX1200-TRUE16-NEXT: v_ldexp_f32 v0, v0, 11
; GFX1200-TRUE16-NEXT: s_cbranch_scc1 .LBB0_6
; GFX1200-TRUE16-NEXT: .LBB0_7: ; %frem.loop_exit
-; GFX1200-TRUE16-NEXT: ; implicit-def: $vgpr2
+; GFX1200-TRUE16-NEXT: ; implicit-def: $vgpr0
; GFX1200-TRUE16-NEXT: .LBB0_8: ; %Flow19
-; GFX1200-TRUE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x7fff, v0
-; GFX1200-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0, v1.l
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-TRUE16-NEXT: v_cmp_nle_f16_e64 s0, 0x7c00, v0.l
-; GFX1200-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v2.l, s0
-; GFX1200-TRUE16-NEXT: global_store_b16 v3, v0, s[8:9]
+; GFX1200-TRUE16-NEXT: s_cmp_lg_f16 s3, 0
+; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1200-TRUE16-NEXT: s_cselect_b32 s3, -1, 0
+; GFX1200-TRUE16-NEXT: s_cmp_nge_f16 s2, 0x7c00
+; GFX1200-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1200-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX1200-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v0.l, s2
+; GFX1200-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1200-TRUE16-NEXT: s_endpgm
;
; GFX1200-FAKE16-LABEL: frem_f16:
; GFX1200-FAKE16: ; %bb.0:
; GFX1200-FAKE16-NEXT: s_clause 0x1
-; GFX1200-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1200-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1200-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX1200-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-FAKE16-NEXT: s_clause 0x1
-; GFX1200-FAKE16-NEXT: global_load_u16 v0, v1, s[10:11]
-; GFX1200-FAKE16-NEXT: global_load_u16 v1, v1, s[0:1] offset:8
+; GFX1200-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
+; GFX1200-FAKE16-NEXT: global_load_u16 v0, v0, s[6:7] offset:8
; GFX1200-FAKE16-NEXT: s_wait_loadcnt 0x1
-; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX1200-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1200-FAKE16-NEXT: s_and_b32 s0, s0, 0x7fff
-; GFX1200-FAKE16-NEXT: s_and_b32 s2, s1, 0x7fff
-; GFX1200-FAKE16-NEXT: s_cvt_f32_f16 s1, s0
-; GFX1200-FAKE16-NEXT: s_cvt_f32_f16 s0, s2
+; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1200-FAKE16-NEXT: s_and_b32 s2, s4, 0x7fff
+; GFX1200-FAKE16-NEXT: s_and_b32 s5, s3, 0x7fff
+; GFX1200-FAKE16-NEXT: s_cvt_f32_f16 s6, s2
+; GFX1200-FAKE16-NEXT: s_cvt_f32_f16 s5, s5
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1200-FAKE16-NEXT: s_cmp_ngt_f32 s1, s0
+; GFX1200-FAKE16-NEXT: s_cmp_ngt_f32 s6, s5
; GFX1200-FAKE16-NEXT: s_cbranch_scc0 .LBB0_2
; GFX1200-FAKE16-NEXT: ; %bb.1: ; %frem.else
-; GFX1200-FAKE16-NEXT: s_cmp_eq_f32 s1, s0
-; GFX1200-FAKE16-NEXT: v_and_b32_e32 v2, 0x8000, v0
-; GFX1200-FAKE16-NEXT: s_mov_b32 s2, 0
+; GFX1200-FAKE16-NEXT: s_and_b32 s7, 0x8000, s4
+; GFX1200-FAKE16-NEXT: s_cmp_eq_f32 s6, s5
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v0, s7
+; GFX1200-FAKE16-NEXT: s_mov_b32 s7, 0
; GFX1200-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc_lo
+; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v0, s4, v0, vcc_lo
; GFX1200-FAKE16-NEXT: s_branch .LBB0_3
; GFX1200-FAKE16-NEXT: .LBB0_2:
-; GFX1200-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1200-FAKE16-NEXT: ; implicit-def: $vgpr2
+; GFX1200-FAKE16-NEXT: s_mov_b32 s7, -1
+; GFX1200-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1200-FAKE16-NEXT: .LBB0_3: ; %Flow18
-; GFX1200-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1200-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1200-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1200-FAKE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-FAKE16-NEXT: s_and_b32 s7, s7, exec_lo
+; GFX1200-FAKE16-NEXT: s_cselect_b32 s7, 1, 0
+; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-FAKE16-NEXT: s_cmp_lg_u32 s7, 1
; GFX1200-FAKE16-NEXT: s_cbranch_scc1 .LBB0_9
; GFX1200-FAKE16-NEXT: ; %bb.4: ; %frem.compute
-; GFX1200-FAKE16-NEXT: v_frexp_mant_f32_e32 v3, s0
-; GFX1200-FAKE16-NEXT: v_frexp_mant_f32_e32 v2, s1
-; GFX1200-FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v5, s1
+; GFX1200-FAKE16-NEXT: v_frexp_mant_f32_e32 v1, s5
+; GFX1200-FAKE16-NEXT: v_frexp_mant_f32_e32 v0, s6
+; GFX1200-FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v3, s6
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v3, v3, 1
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v4, v2, 11
-; GFX1200-FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v2, s0
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v1, v1, 1
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v2, v0, 11
+; GFX1200-FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v0, s5
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1200-FAKE16-NEXT: v_div_scale_f32 v7, null, v3, v3, 1.0
+; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
+; GFX1200-FAKE16-NEXT: v_div_scale_f32 v5, null, v1, v1, 1.0
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v2, -1, v2
-; GFX1200-FAKE16-NEXT: v_rcp_f32_e32 v8, v7
+; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v0, -1, v0
+; GFX1200-FAKE16-NEXT: v_rcp_f32_e32 v6, v5
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_not_b32_e32 v6, v2
-; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v6, v6, v5
-; GFX1200-FAKE16-NEXT: v_div_scale_f32 v5, vcc_lo, 1.0, v3, 1.0
+; GFX1200-FAKE16-NEXT: v_not_b32_e32 v4, v0
+; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v4, v4, v3
+; GFX1200-FAKE16-NEXT: v_div_scale_f32 v3, vcc_lo, 1.0, v1, 1.0
; GFX1200-FAKE16-NEXT: s_denorm_mode 15
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_fma_f32 v9, -v7, v8, 1.0
-; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v8, v9, v8
+; GFX1200-FAKE16-NEXT: v_fma_f32 v7, -v5, v6, 1.0
+; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v6, v7, v6
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v9, v5, v8
-; GFX1200-FAKE16-NEXT: v_fma_f32 v10, -v7, v9, v5
+; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v7, v3, v6
+; GFX1200-FAKE16-NEXT: v_fma_f32 v8, -v5, v7, v3
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v9, v10, v8
-; GFX1200-FAKE16-NEXT: v_fma_f32 v5, -v7, v9, v5
+; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v7, v8, v6
+; GFX1200-FAKE16-NEXT: v_fma_f32 v3, -v5, v7, v3
; GFX1200-FAKE16-NEXT: s_denorm_mode 12
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1200-FAKE16-NEXT: v_div_fmas_f32 v5, v5, v8, v9
-; GFX1200-FAKE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v6
-; GFX1200-FAKE16-NEXT: v_div_fixup_f32 v5, v5, v3, 1.0
+; GFX1200-FAKE16-NEXT: v_div_fmas_f32 v3, v3, v6, v7
+; GFX1200-FAKE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v4
+; GFX1200-FAKE16-NEXT: v_div_fixup_f32 v3, v3, v1, 1.0
; GFX1200-FAKE16-NEXT: s_cbranch_vccnz .LBB0_8
; GFX1200-FAKE16-NEXT: ; %bb.5: ; %frem.loop_body.preheader
-; GFX1200-FAKE16-NEXT: s_sub_co_i32 s0, s1, s0
+; GFX1200-FAKE16-NEXT: s_sub_co_i32 s5, s6, s5
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: s_add_co_i32 s0, s0, 11
+; GFX1200-FAKE16-NEXT: s_add_co_i32 s5, s5, 11
; GFX1200-FAKE16-NEXT: .LBB0_6: ; %frem.loop_body
; GFX1200-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v7, v4
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v5, v2
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: s_add_co_i32 s0, s0, -11
+; GFX1200-FAKE16-NEXT: s_add_co_i32 s5, s5, -11
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: s_cmp_gt_i32 s0, 11
-; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v4, v7, v5
+; GFX1200-FAKE16-NEXT: s_cmp_gt_i32 s5, 11
+; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v2, v5, v3
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_rndne_f32_e32 v4, v4
-; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v4, 0x80000000, v4
+; GFX1200-FAKE16-NEXT: v_rndne_f32_e32 v2, v2
+; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_fma_f32 v4, v4, v3, v7
-; GFX1200-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v4
-; GFX1200-FAKE16-NEXT: v_add_f32_e32 v6, v4, v3
+; GFX1200-FAKE16-NEXT: v_fma_f32 v2, v2, v1, v5
+; GFX1200-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v2
+; GFX1200-FAKE16-NEXT: v_add_f32_e32 v4, v2, v1
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v4, v4, 11
+; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v2, v2, 11
; GFX1200-FAKE16-NEXT: s_cbranch_scc1 .LBB0_6
; GFX1200-FAKE16-NEXT: ; %bb.7: ; %Flow
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v6, s0
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v4, v7
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v4, s5
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v2, v5
; GFX1200-FAKE16-NEXT: .LBB0_8: ; %frem.loop_exit
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v6, -10, v6
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v4, v4, v6
+; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v4, -10, v4
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v2, v2, v4
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v5, v4, v5
-; GFX1200-FAKE16-NEXT: v_rndne_f32_e32 v5, v5
+; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v3, v2, v3
+; GFX1200-FAKE16-NEXT: v_rndne_f32_e32 v3, v3
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
-; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v4, v5, v3
+; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
+; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v2, v3, v1
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v4
-; GFX1200-FAKE16-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX1200-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v2
+; GFX1200-FAKE16-NEXT: v_add_f32_e32 v1, v2, v1
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo
+; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc_lo
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v2, v3, v2
-; GFX1200-FAKE16-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v0, v1, v0
+; GFX1200-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_bfi_b32 v2, 0x7fff, v2, v0
+; GFX1200-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, s4
; GFX1200-FAKE16-NEXT: .LBB0_9: ; %Flow19
-; GFX1200-FAKE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x7fff, v0
-; GFX1200-FAKE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0, v1
-; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-FAKE16-NEXT: v_cmp_nle_f16_e64 s0, 0x7c00, v0
-; GFX1200-FAKE16-NEXT: s_and_b32 vcc_lo, s0, vcc_lo
+; GFX1200-FAKE16-NEXT: s_cmp_lg_f16 s3, 0
+; GFX1200-FAKE16-NEXT: s_cselect_b32 s3, -1, 0
+; GFX1200-FAKE16-NEXT: s_cmp_nge_f16 s2, 0x7c00
+; GFX1200-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1200-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1200-FAKE16-NEXT: s_and_b32 vcc_lo, s2, s3
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
-; GFX1200-FAKE16-NEXT: global_store_b16 v3, v0, s[8:9]
+; GFX1200-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_cndmask_b32 v0, 0x7e00, v0
+; GFX1200-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1200-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %in2) #0 {
%gep2 = getelementptr half, ptr addrspace(1) %in2, i32 4
@@ -1635,34 +1640,35 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1200-TRUE16-NEXT: s_clause 0x1
; GFX1200-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1200-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v2, 0
; GFX1200-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v3, v1, s[4:5] offset:8
+; GFX1200-TRUE16-NEXT: global_load_u16 v3, v2, s[2:3]
+; GFX1200-TRUE16-NEXT: global_load_u16 v4, v2, s[4:5] offset:8
; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v3.l
+; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v4.l
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v0, v0
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v4, v2, v0, neg(0) op_sel_hi:[1,0,0]
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v1, v3, v0, neg(0) op_sel_hi:[1,0,0]
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v4, v5, v0
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v4, v1, v3 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v1, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v4, v1, v3 op_sel_hi:[1,0,1]
; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1200-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
+; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v4.l, v1.l
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v3.l
-; GFX1200-TRUE16-NEXT: global_store_b16 v1, v2, s[0:1]
+; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v3.l, v0.l, v4.l
+; GFX1200-TRUE16-NEXT: global_store_b16 v2, v3, s[0:1]
; GFX1200-TRUE16-NEXT: s_endpgm
;
; GFX1200-FAKE16-LABEL: fast_frem_f16:
@@ -2118,34 +2124,35 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX1200-TRUE16-NEXT: s_clause 0x1
; GFX1200-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1200-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v2, 0
; GFX1200-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v3, v1, s[4:5] offset:8
+; GFX1200-TRUE16-NEXT: global_load_u16 v3, v2, s[2:3]
+; GFX1200-TRUE16-NEXT: global_load_u16 v4, v2, s[4:5] offset:8
; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v3.l
+; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v4.l
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v0, v0
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v4, v2, v0, neg(0) op_sel_hi:[1,0,0]
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v1, v3, v0, neg(0) op_sel_hi:[1,0,0]
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v4, v5, v0
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v4, v1, v3 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v1, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v4, v1, v3 op_sel_hi:[1,0,1]
; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1200-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
+; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v4.l, v1.l
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v3.l
-; GFX1200-TRUE16-NEXT: global_store_b16 v1, v2, s[0:1]
+; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v3.l, v0.l, v4.l
+; GFX1200-TRUE16-NEXT: global_store_b16 v2, v3, s[0:1]
; GFX1200-TRUE16-NEXT: s_endpgm
;
; GFX1200-FAKE16-LABEL: unsafe_frem_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll
index 9a569fd127d73a..14886857fa2f08 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll
@@ -102,7 +102,7 @@ define amdgpu_kernel void @cos_f16(ptr addrspace(1) %r, ptr addrspace(1) %a) {
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mul_f16_e32 v0.l, 0.15915494, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll
index ad690fe234bc79..1c035c4709babe 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll
@@ -96,7 +96,7 @@ define amdgpu_kernel void @rint_f16(
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_rndne_f16_e32 v0.l, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll
index d8f53af61b3431..0a5df29922f452 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll
@@ -102,7 +102,7 @@ define amdgpu_kernel void @sin_f16(ptr addrspace(1) %r, ptr addrspace(1) %a) {
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mul_f16_e32 v0.l, 0.15915494, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
index 30d966eb1b7c2f..14945be58876a2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
@@ -23,7 +23,7 @@ define amdgpu_kernel void @sqrt_bf16(ptr addrspace(1) %r, ptr addrspace(1) %a) {
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s8, s2
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s9, s3
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-SDAG-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
+; GFX12-TRUE16-SDAG-NEXT: buffer_load_u16 v0, off, s[8:11], null
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s5, s1
; GFX12-TRUE16-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-SDAG-NEXT: v_sqrt_bf16_e32 v0.l, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
index bcaba060b812fa..c9f4e1993cd736 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
@@ -129,7 +129,7 @@ define amdgpu_kernel void @sqrt_f16(
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s8, s2
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s9, s3
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-SDAG-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
+; GFX12-TRUE16-SDAG-NEXT: buffer_load_u16 v0, off, s[8:11], null
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s5, s1
; GFX12-TRUE16-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-SDAG-NEXT: v_sqrt_f16_e32 v0.l, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll
index 4664a4475d22ac..9715cb0e3ee5b8 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll
@@ -95,7 +95,7 @@ define amdgpu_kernel void @trunc_f16(
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index 17bb36e3d943e4..9c64684bf723f2 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -534,55 +534,29 @@ define amdgpu_kernel void @constant_load_v16i1(ptr addrspace(1) %out, ptr addrsp
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_load_v16i1:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_load_v16i1:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-TRUE16-LABEL: constant_load_v16i1:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-TRUE16-NEXT: v_nop
-; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1250-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1250-TRUE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_load_v16i1:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
;
-; GFX1250-FAKE16-LABEL: constant_load_v16i1:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-FAKE16-NEXT: v_nop
-; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; GFX1250-FAKE16-NEXT: s_endpgm
+; GFX1250-LABEL: constant_load_v16i1:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
store <16 x i1> %load, ptr addrspace(1) %out
ret void
@@ -2240,32 +2214,31 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s6, 0xffff, s2
; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10001
; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10007
-; GFX12-NEXT: s_bfe_u32 s7, s2, 0x10009
-; GFX12-NEXT: s_bfe_u32 s8, s2, 0x1000d
-; GFX12-NEXT: s_and_b32 s9, s2, 1
-; GFX12-NEXT: s_bfe_u32 s10, s2, 0x1000a
-; GFX12-NEXT: s_bfe_u32 s2, s2, 0x1000c
-; GFX12-NEXT: s_bfe_u32 s11, s6, 0x10005
-; GFX12-NEXT: s_bfe_u32 s12, s6, 0x1000b
-; GFX12-NEXT: s_lshr_b32 s13, s6, 15
-; GFX12-NEXT: s_bfe_u32 s14, s6, 0x10002
-; GFX12-NEXT: s_bfe_u32 s15, s6, 0x10006
-; GFX12-NEXT: s_bfe_u32 s16, s6, 0x10004
-; GFX12-NEXT: s_bfe_u32 s17, s6, 0x10008
-; GFX12-NEXT: s_bfe_u32 s6, s6, 0x1000e
-; GFX12-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v1, s8
-; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s7
-; GFX12-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v2, s6
-; GFX12-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v4, s17
-; GFX12-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s5
-; GFX12-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v8, s16
-; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s15
-; GFX12-NEXT: v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v13, s4
-; GFX12-NEXT: v_mov_b32_e32 v14, s14
+; GFX12-NEXT: s_bfe_u32 s6, s2, 0x10005
+; GFX12-NEXT: s_bfe_u32 s7, s2, 0x1000b
+; GFX12-NEXT: s_bfe_u32 s8, s2, 0x10009
+; GFX12-NEXT: s_lshr_b32 s9, s2, 15
+; GFX12-NEXT: s_bfe_u32 s10, s2, 0x1000d
+; GFX12-NEXT: s_and_b32 s11, s2, 1
+; GFX12-NEXT: s_bfe_u32 s12, s2, 0x10002
+; GFX12-NEXT: s_bfe_u32 s13, s2, 0x10006
+; GFX12-NEXT: s_bfe_u32 s14, s2, 0x10004
+; GFX12-NEXT: s_bfe_u32 s15, s2, 0x1000a
+; GFX12-NEXT: s_bfe_u32 s16, s2, 0x10008
+; GFX12-NEXT: s_bfe_u32 s17, s2, 0x1000c
+; GFX12-NEXT: s_bfe_u32 s2, s2, 0x1000e
+; GFX12-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v1, s10
+; GFX12-NEXT: v_dual_mov_b32 v0, s17 :: v_dual_mov_b32 v3, s9
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v5, s8
+; GFX12-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v7, s7
+; GFX12-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v9, s6
+; GFX12-NEXT: v_dual_mov_b32 v8, s14 :: v_dual_mov_b32 v11, s5
+; GFX12-NEXT: v_dual_mov_b32 v10, s13 :: v_dual_mov_b32 v13, s4
+; GFX12-NEXT: v_dual_mov_b32 v12, s11 :: v_dual_mov_b32 v15, s3
+; GFX12-NEXT: v_mov_b32_e32 v14, s12
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX12-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
@@ -2283,32 +2256,31 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_and_b32 s6, 0xffff, s2
; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10001
; GFX1250-NEXT: s_bfe_u32 s5, s2, 0x10007
-; GFX1250-NEXT: s_bfe_u32 s7, s2, 0x10009
-; GFX1250-NEXT: s_bfe_u32 s8, s2, 0x1000d
-; GFX1250-NEXT: s_and_b32 s9, s2, 1
-; GFX1250-NEXT: s_bfe_u32 s10, s2, 0x1000a
-; GFX1250-NEXT: s_bfe_u32 s2, s2, 0x1000c
-; GFX1250-NEXT: s_bfe_u32 s11, s6, 0x10005
-; GFX1250-NEXT: s_bfe_u32 s12, s6, 0x1000b
-; GFX1250-NEXT: s_lshr_b32 s13, s6, 15
-; GFX1250-NEXT: s_bfe_u32 s14, s6, 0x10002
-; GFX1250-NEXT: s_bfe_u32 s15, s6, 0x10006
-; GFX1250-NEXT: s_bfe_u32 s16, s6, 0x10004
-; GFX1250-NEXT: s_bfe_u32 s17, s6, 0x10008
-; GFX1250-NEXT: s_bfe_u32 s6, s6, 0x1000e
-; GFX1250-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v0, s2
-; GFX1250-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v5, s7
-; GFX1250-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v2, s6
-; GFX1250-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v4, s17
-; GFX1250-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s5
-; GFX1250-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v8, s16
-; GFX1250-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s15
-; GFX1250-NEXT: v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v13, s4
-; GFX1250-NEXT: v_mov_b32_e32 v14, s14
+; GFX1250-NEXT: s_bfe_u32 s6, s2, 0x10005
+; GFX1250-NEXT: s_bfe_u32 s7, s2, 0x1000b
+; GFX1250-NEXT: s_bfe_u32 s8, s2, 0x10009
+; GFX1250-NEXT: s_lshr_b32 s9, s2, 15
+; GFX1250-NEXT: s_bfe_u32 s10, s2, 0x1000d
+; GFX1250-NEXT: s_and_b32 s11, s2, 1
+; GFX1250-NEXT: s_bfe_u32 s12, s2, 0x10002
+; GFX1250-NEXT: s_bfe_u32 s13, s2, 0x10006
+; GFX1250-NEXT: s_bfe_u32 s14, s2, 0x10004
+; GFX1250-NEXT: s_bfe_u32 s15, s2, 0x1000a
+; GFX1250-NEXT: s_bfe_u32 s16, s2, 0x10008
+; GFX1250-NEXT: s_bfe_u32 s17, s2, 0x1000c
+; GFX1250-NEXT: s_bfe_u32 s2, s2, 0x1000e
+; GFX1250-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v0, s17
+; GFX1250-NEXT: v_dual_mov_b32 v1, s10 :: v_dual_mov_b32 v2, s2
+; GFX1250-NEXT: v_dual_mov_b32 v3, s9 :: v_dual_mov_b32 v4, s16
+; GFX1250-NEXT: v_dual_mov_b32 v5, s8 :: v_dual_mov_b32 v6, s15
+; GFX1250-NEXT: v_dual_mov_b32 v7, s7 :: v_dual_mov_b32 v8, s14
+; GFX1250-NEXT: v_dual_mov_b32 v9, s6 :: v_dual_mov_b32 v10, s13
+; GFX1250-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v12, s11
+; GFX1250-NEXT: v_dual_mov_b32 v13, s4 :: v_dual_mov_b32 v14, s12
+; GFX1250-NEXT: v_mov_b32_e32 v15, s3
; GFX1250-NEXT: s_clause 0x3
; GFX1250-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX1250-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
@@ -7288,38 +7260,36 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000a
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s4
-; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10009
-; GFX12-NEXT: s_bfe_u32 s5, s3, 0x1000b
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, s5
-; GFX12-NEXT: s_bfe_u32 s5, s3, 0x10008
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000a
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s3
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000b
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10009
+; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, v1
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10008
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s5
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_lshr_b32 s4, s3, 15
-; GFX12-NEXT: s_bfe_u32 s5, s3, 0x1000e
-; GFX12-NEXT: s_bfe_u32 s3, s3, 0x10005
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: s_lshr_b32 s3, s2, 15
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000e
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s5
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000d
-; GFX12-NEXT: s_bfe_u32 s5, s2, 0x1000c
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000d
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000c
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:112
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s5
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10007
-; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10006
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10007
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10006
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s5
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10005
; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10004
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7350,51 +7320,47 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000a
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s4
-; GFX1250-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10009
-; GFX1250-NEXT: s_bfe_u32 s5, s3, 0x1000b
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, s5
-; GFX1250-NEXT: s_bfe_u32 s5, s3, 0x10008
-; GFX1250-NEXT: v_mov_b32_e32 v1, v3
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x1000a
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000b
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10009
+; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, v1
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10008
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-NEXT: s_lshr_b32 s4, s3, 15
-; GFX1250-NEXT: s_bfe_u32 s5, s3, 0x1000e
-; GFX1250-NEXT: s_bfe_u32 s3, s3, 0x10005
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:64
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
+; GFX1250-NEXT: s_lshr_b32 s3, s2, 15
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000e
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000d
-; GFX1250-NEXT: s_bfe_u32 s5, s2, 0x1000c
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:112
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x1000d
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000c
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:112
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10007
-; GFX1250-NEXT: s_bfe_u32 s5, s2, 0x10006
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10007
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10006
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10005
; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10004
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX1250-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-NEXT: s_bfe_u32 s2, s2, 0x10001
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:32
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
-; GFX1250-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10001
+; GFX1250-NEXT: s_and_b32 s2, s2, 1
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:16
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v2, s2
-; GFX1250-NEXT: global_store_b128 v3, v[0:3], s[0:1]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
%ext = zext <16 x i1> %load to <16 x i64>
@@ -7682,72 +7648,67 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_load_u16 s3, s[2:3], 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_lshr_b32 s2, s3, 14
-; GFX12-NEXT: s_lshr_b32 s4, s3, 15
-; GFX12-NEXT: s_lshr_b32 s6, s3, 12
-; GFX12-NEXT: s_lshr_b32 s8, s3, 13
-; GFX12-NEXT: s_lshr_b32 s10, s3, 10
-; GFX12-NEXT: s_lshr_b32 s12, s3, 11
-; GFX12-NEXT: s_lshr_b32 s14, s3, 8
-; GFX12-NEXT: s_lshr_b32 s16, s3, 9
-; GFX12-NEXT: s_lshr_b32 s18, s3, 6
-; GFX12-NEXT: s_lshr_b32 s20, s3, 7
-; GFX12-NEXT: s_lshr_b32 s22, s3, 4
-; GFX12-NEXT: s_lshr_b32 s24, s3, 5
-; GFX12-NEXT: s_lshr_b32 s26, s3, 2
-; GFX12-NEXT: s_lshr_b32 s28, s3, 3
-; GFX12-NEXT: s_lshr_b32 s30, s3, 1
+; GFX12-NEXT: s_lshr_b32 s4, s2, 14
+; GFX12-NEXT: s_lshr_b32 s6, s2, 15
+; GFX12-NEXT: s_lshr_b32 s8, s2, 12
+; GFX12-NEXT: s_lshr_b32 s10, s2, 13
; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v12, s3 :: v_dual_mov_b32 v3, s5
-; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v1, s3
-; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s7
-; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v7, s9
-; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX12-NEXT: s_lshr_b32 s12, s2, 10
+; GFX12-NEXT: s_lshr_b32 s14, s2, 11
; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v9, s11
-; GFX12-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v11, s13
-; GFX12-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v1, s5
+; GFX12-NEXT: s_lshr_b32 s16, s2, 8
+; GFX12-NEXT: s_lshr_b32 s18, s2, 9
+; GFX12-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s7
+; GFX12-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v5, s9
+; GFX12-NEXT: s_lshr_b32 s20, s2, 6
+; GFX12-NEXT: s_lshr_b32 s22, s2, 7
; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v7, s11
+; GFX12-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v9, s13
+; GFX12-NEXT: s_lshr_b32 s24, s2, 4
+; GFX12-NEXT: s_lshr_b32 s26, s2, 5
; GFX12-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v8, s10 :: v_dual_mov_b32 v17, s15
-; GFX12-NEXT: v_dual_mov_b32 v10, s12 :: v_dual_mov_b32 v19, s17
-; GFX12-NEXT: v_bfe_i32 v12, v12, 0, 1
-; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
+; GFX12-NEXT: s_lshr_b32 s28, s2, 2
+; GFX12-NEXT: s_lshr_b32 s30, s2, 3
; GFX12-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX12-NEXT: v_mov_b32_e32 v16, s14
-; GFX12-NEXT: v_mov_b32_e32 v18, s16
-; GFX12-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v8, s12 :: v_dual_mov_b32 v11, s15
+; GFX12-NEXT: v_dual_mov_b32 v10, s14 :: v_dual_mov_b32 v13, s17
+; GFX12-NEXT: s_lshr_b32 s34, s2, 1
; GFX12-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v12, s16 :: v_dual_mov_b32 v15, s19
+; GFX12-NEXT: v_mov_b32_e32 v14, s18
+; GFX12-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:112
; GFX12-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:96
-; GFX12-NEXT: v_dual_mov_b32 v0, s18 :: v_dual_mov_b32 v5, s23
-; GFX12-NEXT: v_dual_mov_b32 v1, s19 :: v_dual_mov_b32 v2, s20
-; GFX12-NEXT: v_mov_b32_e32 v7, s25
-; GFX12-NEXT: v_mov_b32_e32 v3, s21
-; GFX12-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v4, s22 :: v_dual_mov_b32 v21, s27
-; GFX12-NEXT: v_dual_mov_b32 v6, s24 :: v_dual_mov_b32 v23, s29
-; GFX12-NEXT: v_dual_mov_b32 v20, s26 :: v_dual_mov_b32 v15, s31
-; GFX12-NEXT: v_mov_b32_e32 v22, s28
-; GFX12-NEXT: v_ashrrev_i32_e32 v13, 31, v12
-; GFX12-NEXT: v_mov_b32_e32 v14, s30
+; GFX12-NEXT: v_dual_mov_b32 v0, s20 :: v_dual_mov_b32 v3, s23
+; GFX12-NEXT: v_dual_mov_b32 v1, s21 :: v_dual_mov_b32 v2, s22
+; GFX12-NEXT: v_mov_b32_e32 v5, s25
+; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v4, s24 :: v_dual_mov_b32 v7, s27
+; GFX12-NEXT: v_dual_mov_b32 v6, s26 :: v_dual_mov_b32 v17, s29
+; GFX12-NEXT: v_dual_mov_b32 v16, s28 :: v_dual_mov_b32 v19, s31
+; GFX12-NEXT: v_dual_mov_b32 v18, s30 :: v_dual_mov_b32 v21, s3
+; GFX12-NEXT: v_dual_mov_b32 v20, s2 :: v_dual_mov_b32 v23, s35
+; GFX12-NEXT: v_mov_b32_e32 v22, s34
; GFX12-NEXT: s_clause 0x5
; GFX12-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:80
-; GFX12-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:64
+; GFX12-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:64
; GFX12-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:48
; GFX12-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:32
-; GFX12-NEXT: global_store_b128 v24, v[20:23], s[0:1] offset:16
-; GFX12-NEXT: global_store_b128 v24, v[12:15], s[0:1]
+; GFX12-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:16
+; GFX12-NEXT: global_store_b128 v24, v[20:23], s[0:1]
; GFX12-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_sextload_v16i1_to_v16i64:
@@ -7758,69 +7719,69 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_load_u16 s3, s[2:3], 0x0 nv
+; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v12, s3
-; GFX1250-NEXT: s_lshr_b32 s2, s3, 14
-; GFX1250-NEXT: s_lshr_b32 s4, s3, 15
-; GFX1250-NEXT: s_lshr_b32 s6, s3, 12
-; GFX1250-NEXT: s_lshr_b32 s8, s3, 13
-; GFX1250-NEXT: s_lshr_b32 s10, s3, 10
-; GFX1250-NEXT: s_lshr_b32 s12, s3, 11
-; GFX1250-NEXT: s_lshr_b32 s14, s3, 8
-; GFX1250-NEXT: s_lshr_b32 s16, s3, 9
-; GFX1250-NEXT: s_lshr_b32 s18, s3, 6
-; GFX1250-NEXT: s_lshr_b32 s20, s3, 7
-; GFX1250-NEXT: s_lshr_b32 s22, s3, 4
-; GFX1250-NEXT: s_lshr_b32 s24, s3, 5
-; GFX1250-NEXT: s_lshr_b32 s26, s3, 2
-; GFX1250-NEXT: s_lshr_b32 s28, s3, 3
-; GFX1250-NEXT: s_lshr_b32 s30, s3, 1
-; GFX1250-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX1250-NEXT: s_lshr_b32 s4, s2, 14
+; GFX1250-NEXT: s_lshr_b32 s6, s2, 15
+; GFX1250-NEXT: s_lshr_b32 s8, s2, 12
+; GFX1250-NEXT: s_lshr_b32 s10, s2, 13
; GFX1250-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
-; GFX1250-NEXT: v_bfe_i32 v12, v12, 0, 1
-; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
-; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX1250-NEXT: s_lshr_b32 s12, s2, 10
+; GFX1250-NEXT: s_lshr_b32 s14, s2, 11
; GFX1250-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v5, s7
-; GFX1250-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v7, s9
-; GFX1250-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v0, s4
+; GFX1250-NEXT: s_lshr_b32 s16, s2, 8
+; GFX1250-NEXT: s_lshr_b32 s18, s2, 9
+; GFX1250-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v2, s6
+; GFX1250-NEXT: v_dual_mov_b32 v3, s7 :: v_dual_mov_b32 v4, s8
+; GFX1250-NEXT: s_lshr_b32 s20, s2, 6
+; GFX1250-NEXT: s_lshr_b32 s22, s2, 7
; GFX1250-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v5, s9 :: v_dual_mov_b32 v6, s10
+; GFX1250-NEXT: v_dual_mov_b32 v7, s11 :: v_dual_mov_b32 v8, s12
+; GFX1250-NEXT: s_lshr_b32 s24, s2, 4
+; GFX1250-NEXT: s_lshr_b32 s26, s2, 5
; GFX1250-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v8, s10 :: v_dual_mov_b32 v9, s11
-; GFX1250-NEXT: v_dual_mov_b32 v10, s12 :: v_dual_mov_b32 v11, s13
-; GFX1250-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
+; GFX1250-NEXT: s_lshr_b32 s28, s2, 2
+; GFX1250-NEXT: s_lshr_b32 s30, s2, 3
; GFX1250-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v16, s14 :: v_dual_mov_b32 v17, s15
-; GFX1250-NEXT: v_dual_mov_b32 v18, s16 :: v_dual_ashrrev_i32 v13, 31, v12
-; GFX1250-NEXT: v_mov_b32_e32 v19, s17
-; GFX1250-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v9, s13 :: v_dual_mov_b32 v10, s14
+; GFX1250-NEXT: v_dual_mov_b32 v11, s15 :: v_dual_mov_b32 v12, s16
+; GFX1250-NEXT: s_lshr_b32 s34, s2, 1
; GFX1250-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v13, s17 :: v_dual_mov_b32 v14, s18
+; GFX1250-NEXT: v_mov_b32_e32 v15, s19
+; GFX1250-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:112
; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:96
; GFX1250-NEXT: s_wait_xcnt 0x1
-; GFX1250-NEXT: v_dual_mov_b32 v0, s18 :: v_dual_mov_b32 v1, s19
-; GFX1250-NEXT: v_dual_mov_b32 v2, s20 :: v_dual_mov_b32 v3, s21
+; GFX1250-NEXT: v_dual_mov_b32 v0, s20 :: v_dual_mov_b32 v1, s21
+; GFX1250-NEXT: v_dual_mov_b32 v2, s22 :: v_dual_mov_b32 v3, s23
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v4, s22
-; GFX1250-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v5, s23 :: v_dual_mov_b32 v6, s24
-; GFX1250-NEXT: v_dual_mov_b32 v7, s25 :: v_dual_mov_b32 v20, s26
-; GFX1250-NEXT: v_dual_mov_b32 v21, s27 :: v_dual_mov_b32 v22, s28
-; GFX1250-NEXT: v_dual_mov_b32 v23, s29 :: v_dual_mov_b32 v14, s30
-; GFX1250-NEXT: v_mov_b32_e32 v15, s31
+; GFX1250-NEXT: v_mov_b32_e32 v4, s24
+; GFX1250-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v5, s25 :: v_dual_mov_b32 v6, s26
+; GFX1250-NEXT: v_dual_mov_b32 v7, s27 :: v_dual_mov_b32 v16, s28
+; GFX1250-NEXT: v_dual_mov_b32 v17, s29 :: v_dual_mov_b32 v18, s30
+; GFX1250-NEXT: v_dual_mov_b32 v19, s31 :: v_dual_mov_b32 v20, s2
+; GFX1250-NEXT: v_dual_mov_b32 v21, s3 :: v_dual_mov_b32 v22, s34
+; GFX1250-NEXT: v_mov_b32_e32 v23, s35
; GFX1250-NEXT: s_clause 0x5
; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:80
-; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:64
+; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:64
; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:48
; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:32
-; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[0:1] offset:16
-; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1]
+; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:16
+; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
%ext = sext <16 x i1> %load to <16 x i64>
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
index b06fff0320b3ef..3a61e48e77a98b 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -79,26 +79,15 @@ define amdgpu_kernel void @constant_load_i16(ptr addrspace(1) %out, ptr addrspac
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_load_i16:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_load_i16:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_load_i16:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
entry:
%ld = load i16, ptr addrspace(4) %in
store i16 %ld, ptr addrspace(1) %out
@@ -760,77 +749,47 @@ define amdgpu_kernel void @constant_load_v16i16_align2(ptr addrspace(4) %ptr0) #
; EG-NEXT: MOV * T2.X, literal.x,
; EG-NEXT: 0(0.000000e+00), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_load_v16i16_align2:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_clause 0x7
-; GFX12-TRUE16-NEXT: s_load_u16 s2, s[0:1], 0xc
-; GFX12-TRUE16-NEXT: s_load_u16 s3, s[0:1], 0x8
-; GFX12-TRUE16-NEXT: s_load_u16 s4, s[0:1], 0x4
-; GFX12-TRUE16-NEXT: s_load_u16 s5, s[0:1], 0x0
-; GFX12-TRUE16-NEXT: s_load_u16 s6, s[0:1], 0x1c
-; GFX12-TRUE16-NEXT: s_load_u16 s7, s[0:1], 0x18
-; GFX12-TRUE16-NEXT: s_load_u16 s8, s[0:1], 0x14
-; GFX12-TRUE16-NEXT: s_load_u16 s9, s[0:1], 0x10
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, s4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, s6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, s7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, s8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, s9
-; GFX12-TRUE16-NEXT: s_clause 0x7
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v7, v8, s[0:1] offset:30
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v6, v8, s[0:1] offset:26
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v5, v8, s[0:1] offset:22
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v4, v8, s[0:1] offset:18
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v1, v8, s[0:1] offset:6
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v0, v8, s[0:1] offset:2
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: global_store_b128 v[0:1], v[4:7], off
-; GFX12-TRUE16-NEXT: global_store_b128 v[0:1], v[0:3], off
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_load_v16i16_align2:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_clause 0x7
-; GFX12-FAKE16-NEXT: s_load_u16 s2, s[0:1], 0xc
-; GFX12-FAKE16-NEXT: s_load_u16 s3, s[0:1], 0x8
-; GFX12-FAKE16-NEXT: s_load_u16 s4, s[0:1], 0x4
-; GFX12-FAKE16-NEXT: s_load_u16 s5, s[0:1], 0x0
-; GFX12-FAKE16-NEXT: s_load_u16 s6, s[0:1], 0x1c
-; GFX12-FAKE16-NEXT: s_load_u16 s7, s[0:1], 0x18
-; GFX12-FAKE16-NEXT: s_load_u16 s8, s[0:1], 0x14
-; GFX12-FAKE16-NEXT: s_load_u16 s9, s[0:1], 0x10
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v3, s2
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v7, s6
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, s7 :: v_dual_mov_b32 v5, s8
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s9
-; GFX12-FAKE16-NEXT: s_clause 0x7
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v7, v8, s[0:1] offset:30
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v6, v8, s[0:1] offset:26
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v5, v8, s[0:1] offset:22
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v4, v8, s[0:1] offset:18
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v1, v8, s[0:1] offset:6
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v0, v8, s[0:1] offset:2
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: global_store_b128 v[0:1], v[4:7], off
-; GFX12-FAKE16-NEXT: global_store_b128 v[0:1], v[0:3], off
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_load_v16i16_align2:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_clause 0xf
+; GFX12-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX12-NEXT: s_load_u16 s3, s[0:1], 0x2
+; GFX12-NEXT: s_load_u16 s4, s[0:1], 0x4
+; GFX12-NEXT: s_load_u16 s5, s[0:1], 0x6
+; GFX12-NEXT: s_load_u16 s6, s[0:1], 0x8
+; GFX12-NEXT: s_load_u16 s7, s[0:1], 0xa
+; GFX12-NEXT: s_load_u16 s8, s[0:1], 0xc
+; GFX12-NEXT: s_load_u16 s9, s[0:1], 0xe
+; GFX12-NEXT: s_load_u16 s10, s[0:1], 0x10
+; GFX12-NEXT: s_load_u16 s11, s[0:1], 0x18
+; GFX12-NEXT: s_load_u16 s12, s[0:1], 0x1a
+; GFX12-NEXT: s_load_u16 s13, s[0:1], 0x1c
+; GFX12-NEXT: s_load_u16 s14, s[0:1], 0x1e
+; GFX12-NEXT: s_load_u16 s15, s[0:1], 0x12
+; GFX12-NEXT: s_load_u16 s16, s[0:1], 0x14
+; GFX12-NEXT: s_load_u16 s0, s[0:1], 0x16
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX12-NEXT: s_pack_ll_b32_b16 s4, s4, s5
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s4
+; GFX12-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX12-NEXT: s_pack_ll_b32_b16 s1, s8, s9
+; GFX12-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s1
+; GFX12-NEXT: s_pack_ll_b32_b16 s5, s11, s12
+; GFX12-NEXT: s_pack_ll_b32_b16 s3, s13, s14
+; GFX12-NEXT: s_pack_ll_b32_b16 s7, s10, s15
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s16, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_dual_mov_b32 v0, s7 :: v_dual_mov_b32 v1, s0
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_store_b128 v[0:1], v[0:3], off
+; GFX12-NEXT: global_store_b128 v[0:1], v[4:7], off
+; GFX12-NEXT: s_endpgm
entry:
%ld = load <16 x i16>, ptr addrspace(4) %ptr0, align 2
store <16 x i16> %ld, ptr addrspace(1) poison, align 32
@@ -5430,8 +5389,6 @@ define amdgpu_kernel void @constant_zextload_i16_to_i64(ptr addrspace(1) %out, p
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX12-NEXT: global_store_b64 v1, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
@@ -5522,9 +5479,8 @@ define amdgpu_kernel void @constant_sextload_i16_to_i64(ptr addrspace(1) %out, p
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_sext_i32_i16 s2, s2
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_ashr_i32 s3, s2, 31
+; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x100000
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT: v_mov_b32_e32 v0, s2
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
@@ -5609,8 +5565,6 @@ define amdgpu_kernel void @constant_zextload_v1i16_to_v1i64(ptr addrspace(1) %ou
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX12-NEXT: global_store_b64 v1, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
@@ -5696,9 +5650,8 @@ define amdgpu_kernel void @constant_sextload_v1i16_to_v1i64(ptr addrspace(1) %ou
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_sext_i32_i16 s2, s2
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_ashr_i32 s3, s2, 31
+; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x100000
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT: v_mov_b32_e32 v0, s2
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
@@ -9007,3 +8960,6 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) %
; }
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-FAKE16: {{.*}}
+; GFX12-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
index 20d5d812779b8d..c1fa5ed6073d0d 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
@@ -178,26 +178,15 @@ define amdgpu_kernel void @constant_load_v2i8(ptr addrspace(1) %out, ptr addrspa
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_load_v2i8:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_load_v2i8:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_load_v2i8:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
entry:
%ld = load <2 x i8>, ptr addrspace(4) %in
store <2 x i8> %ld, ptr addrspace(1) %out
@@ -951,12 +940,11 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i32(ptr addrspace(1) %out
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX12-NEXT: s_and_b32 s2, s2, 0xff
-; GFX12-NEXT: s_lshr_b32 s3, s3, 8
+; GFX12-NEXT: s_and_b32 s3, s2, 0xff
+; GFX12-NEXT: s_lshr_b32 s2, s2, 8
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: v_mov_b32_e32 v0, s3
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
@@ -5626,13 +5614,12 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i64(ptr addrspace(1) %out
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX12-NEXT: s_and_b32 s2, s2, 0xff
+; GFX12-NEXT: s_and_b32 s3, s2, 0xff
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
-; GFX12-NEXT: s_lshr_b32 s3, s3, 8
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s3
+; GFX12-NEXT: s_lshr_b32 s2, s2, 8
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, v1
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, v1
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
@@ -5737,20 +5724,15 @@ define amdgpu_kernel void @constant_sextload_v2i8_to_v2i64(ptr addrspace(1) %out
; GFX12-LABEL: constant_sextload_v2i8_to_v2i64:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v4, 0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
-; GFX12-NEXT: s_lshr_b32 s2, s2, 8
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshr_b32 s4, s2, 8
; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x80000
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v3, s3
-; GFX12-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x80000
+; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s5
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
@@ -9466,9 +9448,8 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i16(ptr addrspace(1) %out
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX12-NEXT: s_lshr_b32 s3, s2, 8
; GFX12-NEXT: s_and_b32 s2, s2, 0xff
-; GFX12-NEXT: s_lshr_b32 s3, s3, 8
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_pack_ll_b32_b16 s2, s2, s3
; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index 6c1c11aed600bf..753d0b2ade48a9 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -3324,14 +3324,11 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_load_u16 s0, s[14:15], 0x0
-; GFX1250-NEXT: s_load_u16 s1, s[12:13], 0x0
+; GFX1250-NEXT: s_load_u16 s0, s[12:13], 0x0
+; GFX1250-NEXT: s_load_u16 s1, s[14:15], 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_and_b32 s2, 0xffff, s0
-; GFX1250-NEXT: s_and_b32 s3, 0xffff, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_lt_u32 s3, s2
-; GFX1250-NEXT: s_cselect_b32 s0, s1, s0
+; GFX1250-NEXT: s_cmp_lt_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, s0, s1
; GFX1250-NEXT: s_cselect_b32 s1, 1, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, s0
; GFX1250-NEXT: v_mov_b32_e32 v2, s1
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
index 99b77b1517bd2b..b1184b56f4b79e 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
@@ -9,13 +9,11 @@ define amdgpu_kernel void @exp_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[V_EXP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_EXP_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[V_EXP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_EXP_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_EXP_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_EXP_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
@@ -32,13 +30,11 @@ define amdgpu_kernel void @log_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[V_LOG_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_LOG_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[V_LOG_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_LOG_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_LOG_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_LOG_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
@@ -55,13 +51,11 @@ define amdgpu_kernel void @rcp_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[V_RCP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RCP_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[V_RCP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RCP_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RCP_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RCP_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
@@ -78,13 +72,11 @@ define amdgpu_kernel void @rsq_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[V_RSQ_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RSQ_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[V_RSQ_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RSQ_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RSQ_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RSQ_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
@@ -101,13 +93,11 @@ define amdgpu_kernel void @sqrt_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[V_SQRT_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_SQRT_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[V_SQRT_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_SQRT_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SQRT_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SQRT_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
diff --git a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
index d8ce9f1726965a..bb9dd7fce9c5e1 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=amdgpu9.42--amdhsa < %s | FileCheck -check-prefixes=GFX942 %s
; RUN: llc -mtriple=amdgpu9.0a--amdhsa < %s | FileCheck -check-prefixes=GFX90a %s
-; RUN: llc -mtriple=amdgpu12.50--amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
-; RUN: llc -mtriple=amdgpu12.50--amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+; RUN: llc -mtriple=amdgpu12.50--amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250 %s
+; RUN: llc -mtriple=amdgpu12.50--amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250 %s
define amdgpu_kernel void @preload_block_count_x(ptr addrspace(1) inreg %out) #0 {
; GFX942-LABEL: preload_block_count_x:
@@ -298,30 +298,17 @@ define amdgpu_kernel void @incorrect_type_i16_block_count_x(ptr addrspace(1) inr
; GFX90a-NEXT: global_store_short v0, v1, s[8:9]
; GFX90a-NEXT: s_endpgm
;
-; GFX1250-FAKE16-LABEL: incorrect_type_i16_block_count_x:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-FAKE16-NEXT: v_nop
-; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-FAKE16-NEXT: s_load_u16 s0, s[0:1], 0x8 nv
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX1250-FAKE16-NEXT: global_store_b16 v0, v1, s[2:3]
-; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: incorrect_type_i16_block_count_x:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-REAL16-NEXT: s_load_u16 s0, s[0:1], 0x8 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX1250-REAL16-NEXT: global_store_b16 v1, v0, s[2:3]
-; GFX1250-REAL16-NEXT: s_endpgm
+; GFX1250-LABEL: incorrect_type_i16_block_count_x:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_u16 s0, s[0:1], 0x8 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT: global_store_b16 v0, v1, s[2:3]
+; GFX1250-NEXT: s_endpgm
%imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
%load = load i16, ptr addrspace(4) %imp_arg_ptr
store i16 %load, ptr addrspace(1) %out
More information about the llvm-commits
mailing list