[llvm] [AMDGPU] Legalize uniform 16-bit loads to match subword load instructions (PR #225097)
via llvm-commits
llvm-commits at lists.llvm.org
Tue Sep 29 08:00:30 PDT 2026
https://github.com/LU-JOHN updated https://github.com/llvm/llvm-project/pull/225097
>From a622d4bd22ec65980a3bf1469f901a732778368c Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Mon, 21 Sep 2026 09:30:03 -0500
Subject: [PATCH 1/6] Enable scalar subword loads for plain 16-bit loads
Signed-off-by: John Lu <John.Lu at amd.com>
---
llvm/lib/Target/AMDGPU/SMInstructions.td | 21 +-
.../test/CodeGen/AMDGPU/fcanonicalize.bf16.ll | 50 +-
.../CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll | 42 +-
.../CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll | 42 +-
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 539 +++++++-----------
llvm/test/CodeGen/AMDGPU/load-constant-i16.ll | 217 +++----
llvm/test/CodeGen/AMDGPU/load-constant-i8.ll | 238 +++-----
llvm/test/CodeGen/AMDGPU/min.ll | 22 +-
.../AMDGPU/preload-implicit-kernargs.ll | 38 +-
9 files changed, 477 insertions(+), 732 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index 19aeafe9b30cc..0f155fd898e98 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -980,7 +980,7 @@ multiclass SMLoad_Pattern <string Instr, ValueType vt, bit immci = true> {
}
}
-multiclass ScalarLoadWithExtensionPat <string Instr, SDPatternOperator node, ValueType vt> {
+multiclass ScalarSubwordLoadPat <string Instr, SDPatternOperator node, ValueType vt> {
// 1. IMM offset
def : GCNPat <
(node (SMRDImm i64:$sbase, i32:$offset)),
@@ -1041,12 +1041,19 @@ multiclass ScalarBufferLoadIntrinsicPat <SDPatternOperator name, string Instr> {
// selector to prefer those.
let AddedComplexity = 100 in {
-defm : ScalarLoadWithExtensionPat <"S_LOAD_U8", smrd_extloadi8, i32>;
-defm : ScalarLoadWithExtensionPat <"S_LOAD_U8", smrd_zextloadi8, i32>;
-defm : ScalarLoadWithExtensionPat <"S_LOAD_I8", smrd_sextloadi8, i32>;
-defm : ScalarLoadWithExtensionPat <"S_LOAD_U16", smrd_extloadi16, i32>;
-defm : ScalarLoadWithExtensionPat <"S_LOAD_U16", smrd_zextloadi16, i32>;
-defm : ScalarLoadWithExtensionPat <"S_LOAD_I16", smrd_sextloadi16, i32>;
+// Extending subword loads
+defm : ScalarSubwordLoadPat <"S_LOAD_U8", smrd_extloadi8, i32>;
+defm : ScalarSubwordLoadPat <"S_LOAD_U8", smrd_zextloadi8, i32>;
+defm : ScalarSubwordLoadPat <"S_LOAD_I8", smrd_sextloadi8, i32>;
+defm : ScalarSubwordLoadPat <"S_LOAD_U16", smrd_extloadi16, i32>;
+defm : ScalarSubwordLoadPat <"S_LOAD_U16", smrd_zextloadi16, i32>;
+defm : ScalarSubwordLoadPat <"S_LOAD_I16", smrd_sextloadi16, i32>;
+
+// Non-extending subword loads
+foreach vt = Reg16Types.types in {
+ defm : ScalarSubwordLoadPat <"S_LOAD_U16", smrd_load, vt>;
+}
+
defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_byte, "S_BUFFER_LOAD_I8">;
defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_ubyte, "S_BUFFER_LOAD_U8">;
defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_short, "S_BUFFER_LOAD_I16">;
diff --git a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
index f559943070dd7..6ad7990507113 100644
--- a/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcanonicalize.bf16.ll
@@ -120,11 +120,11 @@ define amdgpu_kernel void @v_test_canonicalize_fabs_var_bf16(ptr addrspace(1) %o
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v1, 0x7fff, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_and_b32 s2, s2, 0x7fff
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, s2 op_sel_hi:[0,1]
; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
@@ -145,11 +145,11 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_fabs_var_bf16(ptr addrspace(
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_or_b32_e32 v1, 0x8000, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_bitset1_b32 s2, 15
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, s2 op_sel_hi:[0,1]
; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
@@ -170,11 +170,11 @@ define amdgpu_kernel void @v_test_canonicalize_fneg_var_bf16(ptr addrspace(1) %o
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v1, 0x8000, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_xor_b32 s2, s2, 0x8000
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, s2 op_sel_hi:[0,1]
; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
@@ -194,11 +194,11 @@ define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_var_bf16(ptr ad
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_xor_b32_e32 v1, 0x8000, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_xor_b32 s2, s2, 0x8000
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, s2 op_sel_hi:[0,1]
; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
@@ -218,11 +218,11 @@ define amdgpu_kernel void @v_test_no_denormals_canonicalize_fneg_fabs_var_bf16(p
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1]
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_or_b32_e32 v1, 0x8000, v1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, v1 op_sel_hi:[0,1]
+; GFX1250-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_bitset1_b32 s2, 15
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_pk_mul_bf16 v1, 1.0, s2 op_sel_hi:[0,1]
; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX1250-NEXT: s_endpgm
%val = load bfloat, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll
index 73f8df5ed6515..73304ba2b319d 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rcp.f16.ll
@@ -65,37 +65,31 @@ define amdgpu_kernel void @rcp_f16(
; GFX12-TRUE16-LABEL: rcp_f16:
; GFX12-TRUE16: ; %bb.0: ; %entry
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX12-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-TRUE16-NEXT: s_mov_b32 s10, s6
-; GFX12-TRUE16-NEXT: s_mov_b32 s11, s7
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
-; GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_rcp_f16_e32 v0.l, v0.l
-; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_s_rcp_f16 s2, s2
+; GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX12-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-TRUE16-NEXT: s_endpgm
;
; GFX12-FAKE16-LABEL: rcp_f16:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_mov_b32 s6, -1
-; GFX12-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; GFX12-FAKE16-NEXT: s_mov_b32 s10, s6
-; GFX12-FAKE16-NEXT: s_mov_b32 s11, s7
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_mov_b32 s8, s2
-; GFX12-FAKE16-NEXT: s_mov_b32 s9, s3
-; GFX12-FAKE16-NEXT: s_mov_b32 s4, s0
-; GFX12-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
-; GFX12-FAKE16-NEXT: s_mov_b32 s5, s1
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_rcp_f16_e32 v0, v0
-; GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_s_rcp_f16 s2, s2
+; GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX12-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %r,
ptr addrspace(1) %a) {
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
index e5979d8c99b60..fd36da8c635bf 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.rsq.f16.ll
@@ -70,37 +70,31 @@ define amdgpu_kernel void @rsq_f16(
; SDAG-GFX12-TRUE16-LABEL: rsq_f16:
; SDAG-GFX12-TRUE16: ; %bb.0: ; %entry
; SDAG-GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s6, -1
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s7, 0x31016000
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s10, s6
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s11, s7
; SDAG-GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; SDAG-GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
-; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
-; SDAG-GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; SDAG-GFX12-TRUE16-NEXT: v_rsq_f16_e32 v0.l, v0.l
-; SDAG-GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; SDAG-GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s3, 0x31016000
+; SDAG-GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; SDAG-GFX12-TRUE16-NEXT: v_s_rsq_f16 s2, s2
+; SDAG-GFX12-TRUE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; SDAG-GFX12-TRUE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; SDAG-GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; SDAG-GFX12-TRUE16-NEXT: s_mov_b32 s2, -1
+; SDAG-GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; SDAG-GFX12-TRUE16-NEXT: s_endpgm
;
; SDAG-GFX12-FAKE16-LABEL: rsq_f16:
; SDAG-GFX12-FAKE16: ; %bb.0: ; %entry
; SDAG-GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s6, -1
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s7, 0x31016000
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s10, s6
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s11, s7
; SDAG-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s8, s2
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s9, s3
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s4, s0
-; SDAG-GFX12-FAKE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
-; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s5, s1
-; SDAG-GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; SDAG-GFX12-FAKE16-NEXT: v_rsq_f16_e32 v0, v0
-; SDAG-GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[4:7], null
+; SDAG-GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s3, 0x31016000
+; SDAG-GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; SDAG-GFX12-FAKE16-NEXT: v_s_rsq_f16 s2, s2
+; SDAG-GFX12-FAKE16-NEXT: s_wait_alu depctr_va_sdst(0)
+; SDAG-GFX12-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1)
+; SDAG-GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, s2
+; SDAG-GFX12-FAKE16-NEXT: s_mov_b32 s2, -1
+; SDAG-GFX12-FAKE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; SDAG-GFX12-FAKE16-NEXT: s_endpgm
;
; GISEL-GCN-LABEL: rsq_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index 050373430cce5..633c54f06a362 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -539,34 +539,50 @@ define amdgpu_kernel void @constant_load_v16i1(ptr addrspace(1) %out, ptr addrsp
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX12-TRUE16-NEXT: s_endpgm
;
; GFX12-FAKE16-LABEL: constant_load_v16i1:
; GFX12-FAKE16: ; %bb.0:
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX12-FAKE16-NEXT: s_endpgm
;
-; GFX1250-LABEL: constant_load_v16i1:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[2:3] nv
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
-; GFX1250-NEXT: s_endpgm
+; GFX1250-TRUE16-LABEL: constant_load_v16i1:
+; GFX1250-TRUE16: ; %bb.0:
+; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-TRUE16-NEXT: v_nop
+; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
+; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
+; GFX1250-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1250-TRUE16-NEXT: s_endpgm
+;
+; GFX1250-FAKE16-LABEL: constant_load_v16i1:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1250-FAKE16-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
store <16 x i1> %load, ptr addrspace(1) %out
ret void
@@ -2218,86 +2234,44 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; EG-NEXT: ADD_INT * T14.X, T11.X, literal.x,
; EG-NEXT: 12(1.681558e-44), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_zextload_v16i1_to_v16i32:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v16, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v16, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-TRUE16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10003
-; GFX12-TRUE16-NEXT: s_bfe_u32 s5, s2, 0x10001
-; GFX12-TRUE16-NEXT: s_bfe_u32 s6, s2, 0x10007
-; GFX12-TRUE16-NEXT: s_bfe_u32 s7, s2, 0x10009
-; GFX12-TRUE16-NEXT: s_bfe_u32 s8, s2, 0x1000d
-; GFX12-TRUE16-NEXT: s_and_b32 s9, s2, 1
-; GFX12-TRUE16-NEXT: s_bfe_u32 s10, s2, 0x1000a
-; GFX12-TRUE16-NEXT: s_bfe_u32 s2, s2, 0x1000c
-; GFX12-TRUE16-NEXT: s_bfe_u32 s11, s3, 0x10005
-; GFX12-TRUE16-NEXT: s_bfe_u32 s12, s3, 0x1000b
-; GFX12-TRUE16-NEXT: s_lshr_b32 s13, s3, 15
-; GFX12-TRUE16-NEXT: s_bfe_u32 s14, s3, 0x10002
-; GFX12-TRUE16-NEXT: s_bfe_u32 s15, s3, 0x10006
-; GFX12-TRUE16-NEXT: s_bfe_u32 s16, s3, 0x10004
-; GFX12-TRUE16-NEXT: s_bfe_u32 s17, s3, 0x10008
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s3, 0x1000e
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v2, s3
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v8, s16
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s10
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v15, s4 :: v_dual_mov_b32 v4, s17
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v10, s15
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v11, s6 :: v_dual_mov_b32 v12, s9
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v14, s14
-; GFX12-TRUE16-NEXT: s_clause 0x3
-; GFX12-TRUE16-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
-; GFX12-TRUE16-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
-; GFX12-TRUE16-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
-; GFX12-TRUE16-NEXT: global_store_b128 v16, v[12:15], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_v16i1_to_v16i32:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v16, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v16, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-FAKE16-NEXT: s_and_b32 s6, 0xffff, s2
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x10001
-; GFX12-FAKE16-NEXT: s_bfe_u32 s5, s2, 0x10007
-; GFX12-FAKE16-NEXT: s_bfe_u32 s7, s2, 0x10009
-; GFX12-FAKE16-NEXT: s_bfe_u32 s8, s2, 0x1000d
-; GFX12-FAKE16-NEXT: s_and_b32 s9, s2, 1
-; GFX12-FAKE16-NEXT: s_bfe_u32 s10, s2, 0x1000a
-; GFX12-FAKE16-NEXT: s_bfe_u32 s2, s2, 0x1000c
-; GFX12-FAKE16-NEXT: s_bfe_u32 s11, s6, 0x10005
-; GFX12-FAKE16-NEXT: s_bfe_u32 s12, s6, 0x1000b
-; GFX12-FAKE16-NEXT: s_lshr_b32 s13, s6, 15
-; GFX12-FAKE16-NEXT: s_bfe_u32 s14, s6, 0x10002
-; GFX12-FAKE16-NEXT: s_bfe_u32 s15, s6, 0x10006
-; GFX12-FAKE16-NEXT: s_bfe_u32 s16, s6, 0x10004
-; GFX12-FAKE16-NEXT: s_bfe_u32 s17, s6, 0x10008
-; GFX12-FAKE16-NEXT: s_bfe_u32 s6, s6, 0x1000e
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v13, s4 :: v_dual_mov_b32 v2, s6
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v8, s16
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s10
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v4, s17
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v10, s15
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v12, s9
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v14, s14
-; GFX12-FAKE16-NEXT: s_clause 0x3
-; GFX12-FAKE16-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
-; GFX12-FAKE16-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
-; GFX12-FAKE16-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
-; GFX12-FAKE16-NEXT: global_store_b128 v16, v[12:15], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_v16i1_to_v16i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_and_b32 s6, 0xffff, s2
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10001
+; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10007
+; GFX12-NEXT: s_bfe_u32 s7, s2, 0x10009
+; GFX12-NEXT: s_bfe_u32 s8, s2, 0x1000d
+; GFX12-NEXT: s_and_b32 s9, s2, 1
+; GFX12-NEXT: s_bfe_u32 s10, s2, 0x1000a
+; GFX12-NEXT: s_bfe_u32 s2, s2, 0x1000c
+; GFX12-NEXT: s_bfe_u32 s11, s6, 0x10005
+; GFX12-NEXT: s_bfe_u32 s12, s6, 0x1000b
+; GFX12-NEXT: s_lshr_b32 s13, s6, 15
+; GFX12-NEXT: s_bfe_u32 s14, s6, 0x10002
+; GFX12-NEXT: s_bfe_u32 s15, s6, 0x10006
+; GFX12-NEXT: s_bfe_u32 s16, s6, 0x10004
+; GFX12-NEXT: s_bfe_u32 s17, s6, 0x10008
+; GFX12-NEXT: s_bfe_u32 s6, s6, 0x1000e
+; GFX12-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v1, s8
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s7
+; GFX12-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v2, s6
+; GFX12-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v4, s17
+; GFX12-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s5
+; GFX12-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v8, s16
+; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s15
+; GFX12-NEXT: v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v13, s4
+; GFX12-NEXT: v_mov_b32_e32 v14, s14
+; GFX12-NEXT: s_clause 0x3
+; GFX12-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
+; GFX12-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
+; GFX12-NEXT: global_store_b128 v16, v[8:11], s[0:1] offset:16
+; GFX12-NEXT: global_store_b128 v16, v[12:15], s[0:1]
+; GFX12-NEXT: s_endpgm
;
; GFX1250-LABEL: constant_zextload_v16i1_to_v16i32:
; GFX1250: ; %bb.0:
@@ -2306,11 +2280,9 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v16, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v0, v16, s[2:3] nv
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readfirstlane_b32 s2, v0
+; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_and_b32 s6, 0xffff, s2
; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10001
@@ -2328,14 +2300,15 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_u32 s16, s6, 0x10004
; GFX1250-NEXT: s_bfe_u32 s17, s6, 0x10008
; GFX1250-NEXT: s_bfe_u32 s6, s6, 0x1000e
-; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
-; GFX1250-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s13
-; GFX1250-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s10
-; GFX1250-NEXT: v_dual_mov_b32 v4, s17 :: v_dual_mov_b32 v7, s12
-; GFX1250-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v12, s9
-; GFX1250-NEXT: v_dual_mov_b32 v8, s16 :: v_dual_mov_b32 v9, s11
-; GFX1250-NEXT: v_dual_mov_b32 v10, s15 :: v_dual_mov_b32 v14, s14
-; GFX1250-NEXT: v_dual_mov_b32 v13, s4 :: v_dual_mov_b32 v15, s3
+; GFX1250-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v0, s2
+; GFX1250-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v5, s7
+; GFX1250-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v2, s6
+; GFX1250-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v4, s17
+; GFX1250-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s5
+; GFX1250-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v8, s16
+; GFX1250-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s15
+; GFX1250-NEXT: v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v13, s4
+; GFX1250-NEXT: v_mov_b32_e32 v14, s14
; GFX1250-NEXT: s_clause 0x3
; GFX1250-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX1250-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
@@ -7315,235 +7288,120 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; EG-NEXT: ADD_INT * T22.X, T15.X, literal.x,
; EG-NEXT: 28(3.923636e-44), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_zextload_v16i1_to_v16i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v5, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v7, v3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v9, v3
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v3, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x1000a
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_bfe_u32 v4, v6, 11, 1
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10009
-; GFX12-TRUE16-NEXT: v_bfe_u32 v0, v6, 8, 1
-; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x1000c
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v8, 15, v6
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x1000d
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10007
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[0:3], s[0:1] offset:64
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10006
-; GFX12-TRUE16-NEXT: v_bfe_u32 v0, v6, 5, 1
-; GFX12-TRUE16-NEXT: v_bfe_u32 v6, v6, 14, 1
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10004
-; GFX12-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, v0
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10001
-; GFX12-TRUE16-NEXT: s_and_b32 s2, s2, 1
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[6:9], s[0:1] offset:112
-; GFX12-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_v16i1_to_v16i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v5, v3
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, v3
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v7, v3
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v9, v3
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v3, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v6, 0xffff, v0
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x1000a
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_bfe_u32 v4, v6, 11, 1
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10009
-; GFX12-FAKE16-NEXT: v_bfe_u32 v0, v6, 8, 1
-; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x1000c
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 15, v6
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x1000d
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10007
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[0:3], s[0:1] offset:64
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x10006
-; GFX12-FAKE16-NEXT: v_bfe_u32 v0, v6, 5, 1
-; GFX12-FAKE16-NEXT: v_bfe_u32 v6, v6, 14, 1
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10004
-; GFX12-FAKE16-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, v0
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10001
-; GFX12-FAKE16-NEXT: s_and_b32 s2, s2, 1
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s3
-; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[6:9], s[0:1] offset:112
-; GFX12-FAKE16-NEXT: global_store_b128 v3, v[2:5], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-TRUE16-LABEL: constant_zextload_v16i1_to_v16i64:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-TRUE16-NEXT: v_nop
-; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v3, 0
-; GFX1250-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, v3
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: global_load_u16 v0, v3, s[2:3] nv
-; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: v_readfirstlane_b32 s2, v0
-; GFX1250-TRUE16-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x1000a
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s3, 0x1000b
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v2, s4
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, s5
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10009
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s3, 0x10008
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s5
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s3, 0x1000e
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, s4
-; GFX1250-TRUE16-NEXT: s_lshr_b32 s4, s3, 15
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s3, s3, 0x10005
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:64
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s5
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s2, 0x1000c
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, s4
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x1000d
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:112
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s5
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s5, s2, 0x10006
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v4, s4
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10007
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10004
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX1250-TRUE16-NEXT: s_bfe_u32 s2, s2, 0x10001
-; GFX1250-TRUE16-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
-; GFX1250-TRUE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v2, s2
-; GFX1250-TRUE16-NEXT: global_store_b128 v3, v[0:3], s[0:1]
-; GFX1250-TRUE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_v16i1_to_v16i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000a
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s4
+; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10009
+; GFX12-NEXT: s_bfe_u32 s5, s3, 0x1000b
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, s5
+; GFX12-NEXT: s_bfe_u32 s5, s3, 0x10008
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s5
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_lshr_b32 s4, s3, 15
+; GFX12-NEXT: s_bfe_u32 s5, s3, 0x1000e
+; GFX12-NEXT: s_bfe_u32 s3, s3, 0x10005
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s5
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000d
+; GFX12-NEXT: s_bfe_u32 s5, s2, 0x1000c
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:112
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s5
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10007
+; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10006
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s5
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10004
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10002
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:32
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10001
+; GFX12-NEXT: s_and_b32 s2, s2, 1
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:16
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1]
+; GFX12-NEXT: s_endpgm
;
-; GFX1250-FAKE16-LABEL: constant_zextload_v16i1_to_v16i64:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-FAKE16-NEXT: v_nop
-; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v5, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v9, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v13, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v15, v1 :: v_dual_mov_b32 v17, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v21, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v23, v1 :: v_dual_mov_b32 v25, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v27, v1 :: v_dual_mov_b32 v29, v1
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: global_load_u16 v12, v1, s[2:3] nv
-; GFX1250-FAKE16-NEXT: s_wait_xcnt 0x0
-; GFX1250-FAKE16-NEXT: s_mov_b32 s2, 0xffff
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v31, v1 :: v_dual_bitop2_b32 v22, s2, v12 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v0, v12, 10, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v6, v12, 9, 1
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v28, 1, v12
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v14, v12, 13, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v2, v22, 11, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v4, v22, 8, 1
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 15, v22
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v8, v22, 14, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v18, v12, 7, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v26, v12, 3, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v30, v12, 1, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v24, v12, 2, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v20, v12, 4, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v16, v12, 6, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v12, v12, 12, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v22, v22, 5, 1
-; GFX1250-FAKE16-NEXT: s_clause 0x7
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[4:7], s[0:1] offset:64
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[8:11], s[0:1] offset:112
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[12:15], s[0:1] offset:96
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[16:19], s[0:1] offset:48
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[20:23], s[0:1] offset:32
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[24:27], s[0:1] offset:16
-; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[28:31], s[0:1]
-; GFX1250-FAKE16-NEXT: s_endpgm
+; GFX1250-LABEL: constant_zextload_v16i1_to_v16i64:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000a
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s4
+; GFX1250-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10009
+; GFX1250-NEXT: s_bfe_u32 s5, s3, 0x1000b
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1250-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, s5
+; GFX1250-NEXT: s_bfe_u32 s5, s3, 0x10008
+; GFX1250-NEXT: v_mov_b32_e32 v1, v3
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
+; GFX1250-NEXT: s_lshr_b32 s4, s3, 15
+; GFX1250-NEXT: s_bfe_u32 s5, s3, 0x1000e
+; GFX1250-NEXT: s_bfe_u32 s3, s3, 0x10005
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:64
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000d
+; GFX1250-NEXT: s_bfe_u32 s5, s2, 0x1000c
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:112
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10007
+; GFX1250-NEXT: s_bfe_u32 s5, s2, 0x10006
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10004
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10003
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10002
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
+; GFX1250-NEXT: s_bfe_u32 s2, s2, 0x10001
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
+; GFX1250-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s2
+; GFX1250-NEXT: global_store_b128 v3, v[0:3], s[0:1]
+; GFX1250-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
%ext = zext <16 x i1> %load to <16 x i64>
store <16 x i64> %ext, ptr addrspace(1) %out
@@ -7825,7 +7683,7 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; EG-NEXT: MOV T20.W, T20.Z,
; EG-NEXT: MOV * T14.W, T14.Z,
; EG-NEXT: 28(3.923636e-44), 0(0.000000e+00)
-;
+
; GFX12-TRUE16-LABEL: constant_sextload_v16i1_to_v16i64:
; GFX12-TRUE16: ; %bb.0:
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
@@ -7967,7 +7825,7 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12-FAKE16-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
; GFX12-FAKE16-NEXT: global_store_b128 v32, v[28:31], s[0:1]
; GFX12-FAKE16-NEXT: s_endpgm
-;
+
; GFX1250-LABEL: constant_sextload_v16i1_to_v16i64:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -7975,7 +7833,6 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-NEXT: v_nop
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-NEXT: v_mov_b32_e32 v32, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_u16 v0, v32, s[2:3] nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
index 48a2db2aca1e4..895dedfff038e 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -84,18 +84,19 @@ define amdgpu_kernel void @constant_load_i16(ptr addrspace(1) %out, ptr addrspac
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX12-TRUE16-NEXT: s_endpgm
;
; GFX12-FAKE16-LABEL: constant_load_i16:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX12-FAKE16-NEXT: s_endpgm
entry:
@@ -765,29 +766,31 @@ define amdgpu_kernel void @constant_load_v16i16_align2(ptr addrspace(4) %ptr0) #
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_clause 0x7
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v3, v8, s[0:1] offset:12
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v7, v8, s[0:1] offset:28
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v6, v8, s[0:1] offset:24
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v5, v8, s[0:1] offset:20
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v4, v8, s[0:1] offset:16
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v2, v8, s[0:1] offset:8
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v1, v8, s[0:1] offset:4
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v8, s[0:1]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-TRUE16-NEXT: s_load_u16 s2, s[0:1], 0xc
+; GFX12-TRUE16-NEXT: s_load_u16 s3, s[0:1], 0x8
+; GFX12-TRUE16-NEXT: s_load_u16 s4, s[0:1], 0x4
+; GFX12-TRUE16-NEXT: s_load_u16 s5, s[0:1], 0x0
+; GFX12-TRUE16-NEXT: s_load_u16 s6, s[0:1], 0x1c
+; GFX12-TRUE16-NEXT: s_load_u16 s7, s[0:1], 0x18
+; GFX12-TRUE16-NEXT: s_load_u16 s8, s[0:1], 0x14
+; GFX12-TRUE16-NEXT: s_load_u16 s9, s[0:1], 0x10
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, s2
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, s3
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, s4
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s5
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, s6
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, s7
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, s8
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, s9
+; GFX12-TRUE16-NEXT: s_clause 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v7, v8, s[0:1] offset:30
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v6, v8, s[0:1] offset:26
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v5, v8, s[0:1] offset:22
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v4, v8, s[0:1] offset:18
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v1, v8, s[0:1] offset:6
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x7
; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v0, v8, s[0:1] offset:2
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_clause 0x1
@@ -798,32 +801,30 @@ define amdgpu_kernel void @constant_load_v16i16_align2(ptr addrspace(4) %ptr0) #
; GFX12-FAKE16-LABEL: constant_load_v16i16_align2:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v8, 0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: s_clause 0x7
-; GFX12-FAKE16-NEXT: global_load_u16 v3, v8, s[0:1] offset:12
-; GFX12-FAKE16-NEXT: global_load_u16 v7, v8, s[0:1] offset:28
-; GFX12-FAKE16-NEXT: global_load_u16 v6, v8, s[0:1] offset:24
-; GFX12-FAKE16-NEXT: global_load_u16 v5, v8, s[0:1] offset:20
-; GFX12-FAKE16-NEXT: global_load_u16 v4, v8, s[0:1] offset:16
-; GFX12-FAKE16-NEXT: global_load_u16 v2, v8, s[0:1] offset:8
-; GFX12-FAKE16-NEXT: global_load_u16 v1, v8, s[0:1] offset:4
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v8, s[0:1]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
+; GFX12-FAKE16-NEXT: s_load_u16 s2, s[0:1], 0xc
+; GFX12-FAKE16-NEXT: s_load_u16 s3, s[0:1], 0x8
+; GFX12-FAKE16-NEXT: s_load_u16 s4, s[0:1], 0x4
+; GFX12-FAKE16-NEXT: s_load_u16 s5, s[0:1], 0x0
+; GFX12-FAKE16-NEXT: s_load_u16 s6, s[0:1], 0x1c
+; GFX12-FAKE16-NEXT: s_load_u16 s7, s[0:1], 0x18
+; GFX12-FAKE16-NEXT: s_load_u16 s8, s[0:1], 0x14
+; GFX12-FAKE16-NEXT: s_load_u16 s9, s[0:1], 0x10
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v3, s2
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v7, s6
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, s7 :: v_dual_mov_b32 v5, s8
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s9
+; GFX12-FAKE16-NEXT: s_clause 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v7, v8, s[0:1] offset:30
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v6, v8, s[0:1] offset:26
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v5, v8, s[0:1] offset:22
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v4, v8, s[0:1] offset:18
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v1, v8, s[0:1] offset:6
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x7
; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v0, v8, s[0:1] offset:2
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: s_clause 0x1
@@ -5422,27 +5423,17 @@ define amdgpu_kernel void @constant_zextload_i16_to_i64(ptr addrspace(1) %out, p
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_zextload_i16_to_i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-TRUE16-NEXT: global_store_b64 v1, v[0:1], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_i16_to_i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v1, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-FAKE16-NEXT: global_store_b64 v1, v[0:1], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_i16_to_i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-NEXT: global_store_b64 v1, v[0:1], s[0:1]
+; GFX12-NEXT: s_endpgm
%a = load i16, ptr addrspace(4) %in
%ext = zext i16 %a to i64
store i64 %ext, ptr addrspace(1) %out
@@ -5524,31 +5515,19 @@ define amdgpu_kernel void @constant_sextload_i16_to_i64(ptr addrspace(1) %out, p
; EG-NEXT: ASHR * T0.Y, PV.X, literal.x,
; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_sextload_i16_to_i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v2, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_sextload_i16_to_i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v2, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_sextload_i16_to_i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_sext_i32_i16 s2, s2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_ashr_i32 s3, s2, 31
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT: s_endpgm
%a = load i16, ptr addrspace(4) %in
%ext = sext i16 %a to i64
store i64 %ext, ptr addrspace(1) %out
@@ -5623,27 +5602,17 @@ define amdgpu_kernel void @constant_zextload_v1i16_to_v1i64(ptr addrspace(1) %ou
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_zextload_v1i16_to_v1i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-TRUE16-NEXT: global_store_b64 v1, v[0:1], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_v1i16_to_v1i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v1, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-FAKE16-NEXT: global_store_b64 v1, v[0:1], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_v1i16_to_v1i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_and_b32 s2, s2, 0xffff
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-NEXT: global_store_b64 v1, v[0:1], s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <1 x i16>, ptr addrspace(4) %in
%ext = zext <1 x i16> %load to <1 x i64>
store <1 x i64> %ext, ptr addrspace(1) %out
@@ -5720,31 +5689,19 @@ define amdgpu_kernel void @constant_sextload_v1i16_to_v1i64(ptr addrspace(1) %ou
; EG-NEXT: ASHR * T0.Y, PV.X, literal.x,
; EG-NEXT: 31(4.344025e-44), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_sextload_v1i16_to_v1i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v2, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_sextload_v1i16_to_v1i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v2, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_sextload_v1i16_to_v1i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_sext_i32_i16 s2, s2
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_ashr_i32 s3, s2, 31
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <1 x i16>, ptr addrspace(4) %in
%ext = sext <1 x i16> %load to <1 x i64>
store <1 x i64> %ext, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
index 57d1aba94566e..ec1444d983dbc 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
@@ -183,18 +183,19 @@ define amdgpu_kernel void @constant_load_v2i8(ptr addrspace(1) %out, ptr addrspa
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX12-TRUE16-NEXT: s_endpgm
;
; GFX12-FAKE16-LABEL: constant_load_v2i8:
; GFX12-FAKE16: ; %bb.0: ; %entry
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
; GFX12-FAKE16-NEXT: s_endpgm
entry:
@@ -944,33 +945,20 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i32(ptr addrspace(1) %out
; EG-NEXT: LSHR * T5.X, KC0[2].Y, literal.y,
; EG-NEXT: 255(3.573311e-43), 2(2.802597e-45)
;
-; GFX12-TRUE16-LABEL: constant_zextload_v2i8_to_v2i32:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v2, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-TRUE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_v2i8_to_v2i32:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v2, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v1
-; GFX12-FAKE16-NEXT: global_store_b64 v2, v[0:1], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_v2i8_to_v2i32:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX12-NEXT: s_and_b32 s2, s2, 0xff
+; GFX12-NEXT: s_lshr_b32 s3, s3, 8
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
%ext = zext <2 x i8> %load to <2 x i32>
store <2 x i32> %ext, ptr addrspace(1) %out
@@ -5633,37 +5621,21 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i64(ptr addrspace(1) %out
; EG-NEXT: LSHR * T5.X, KC0[2].Y, literal.y,
; EG-NEXT: 255(3.573311e-43), 2(2.802597e-45)
;
-; GFX12-TRUE16-LABEL: constant_zextload_v2i8_to_v2i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
-; GFX12-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_v2i8_to_v2i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v1, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
-; GFX12-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_v2i8_to_v2i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX12-NEXT: s_and_b32 s2, s2, 0xff
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
+; GFX12-NEXT: s_lshr_b32 s3, s3, 8
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, v1
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
%ext = zext <2 x i8> %load to <2 x i64>
store <2 x i64> %ext, ptr addrspace(1) %out
@@ -5763,39 +5735,25 @@ define amdgpu_kernel void @constant_sextload_v2i8_to_v2i64(ptr addrspace(1) %out
; EG-NEXT: ASHR * T4.W, PV.Z, literal.y,
; EG-NEXT: 2(2.802597e-45), 31(4.344025e-44)
;
-; GFX12-TRUE16-LABEL: constant_sextload_v2i8_to_v2i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v4, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX12-TRUE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v1, 0, 8
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX12-TRUE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_sextload_v2i8_to_v2i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v4, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 8, v0
-; GFX12-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_bfe_i32 v2, v1, 0, 8
-; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX12-FAKE16-NEXT: global_store_b128 v4, v[0:3], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_sextload_v2i8_to_v2i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: v_mov_b32_e32 v4, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: s_lshr_b32 s2, s2, 8
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x80000
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v3, s3
+; GFX12-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX12-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
%ext = sext <2 x i8> %load to <2 x i64>
store <2 x i64> %ext, ptr addrspace(1) %out
@@ -9513,35 +9471,20 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i16(ptr addrspace(1) %out
; EG-NEXT: LSHR * T6.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_zextload_v2i8_to_v2i16:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v1, v0, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v1
-; GFX12-TRUE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_zextload_v2i8_to_v2i16:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v1
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
-; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX12-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_zextload_v2i8_to_v2i16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX12-NEXT: s_and_b32 s2, s2, 0xff
+; GFX12-NEXT: s_lshr_b32 s3, s3, 8
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
%ext = zext <2 x i8> %load to <2 x i16>
store <2 x i16> %ext, ptr addrspace(1) %out
@@ -9643,37 +9586,20 @@ define amdgpu_kernel void @constant_sextload_v2i8_to_v2i16(ptr addrspace(1) %out
; EG-NEXT: LSHR * T6.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_sextload_v2i8_to_v2i16:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v1, v0, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_bfe_i32 v2, v1, 0, 16
-; GFX12-TRUE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.h, v2.l
-; GFX12-TRUE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_sextload_v2i8_to_v2i16:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_bfe_i32 v2, v1, 0, 16
-; GFX12-FAKE16-NEXT: v_bfe_i32 v1, v1, 0, 8
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
-; GFX12-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-FAKE16-NEXT: v_lshl_or_b32 v1, v2, 16, v1
-; GFX12-FAKE16-NEXT: global_store_b32 v0, v1, s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_sextload_v2i8_to_v2i16:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_sext_i32_i16 s3, s2
+; GFX12-NEXT: s_bfe_i32 s2, s2, 0x80000
+; GFX12-NEXT: s_lshr_b32 s3, s3, 8
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b32 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
%ext = sext <2 x i8> %load to <2 x i16>
store <2 x i16> %ext, ptr addrspace(1) %out
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index ac4ee03db3725..b403c6f3419b4 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -3324,17 +3324,17 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: global_load_u16 v1, v0, s[14:15]
-; GFX1250-NEXT: global_load_u16 v2, v0, s[12:13]
-; GFX1250-NEXT: s_wait_loadcnt 0x1
-; GFX1250-NEXT: v_and_b32_e32 v3, 0xffff, v1
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_and_b32_e32 v4, 0xffff, v2
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_cmp_lt_u32_e32 vcc_lo, v4, v3
-; GFX1250-NEXT: v_cndmask_b32_e32 v1, v1, v2, vcc_lo
-; GFX1250-NEXT: v_cndmask_b32_e64 v2, 0, 1, vcc_lo
+; GFX1250-NEXT: s_load_u16 s0, s[14:15], 0x0
+; GFX1250-NEXT: s_load_u16 s1, s[12:13], 0x0
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_and_b32 s2, 0xffff, s0
+; GFX1250-NEXT: s_and_b32 s3, 0xffff, s1
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1250-NEXT: s_cmp_lt_u32 s3, s2
+; GFX1250-NEXT: s_cselect_b32 s0, s1, s0
+; GFX1250-NEXT: s_cselect_b32 s1, 1, 0
+; GFX1250-NEXT: v_mov_b32_e32 v1, s0
+; GFX1250-NEXT: v_mov_b32_e32 v2, s1
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_b16 v0, v1, s[8:9]
; GFX1250-NEXT: global_store_b8 v0, v2, s[10:11]
diff --git a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
index 60078fde83b56..d8ce9f1726965 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
@@ -298,17 +298,30 @@ define amdgpu_kernel void @incorrect_type_i16_block_count_x(ptr addrspace(1) inr
; GFX90a-NEXT: global_store_short v0, v1, s[8:9]
; GFX90a-NEXT: s_endpgm
;
-; GFX1250-LABEL: incorrect_type_i16_block_count_x:
-; GFX1250: ; %bb.0:
-; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-NEXT: v_nop
-; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: global_load_u16 v1, v0, s[0:1] offset:8 nv
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: global_store_b16 v0, v1, s[2:3]
-; GFX1250-NEXT: s_endpgm
+; GFX1250-FAKE16-LABEL: incorrect_type_i16_block_count_x:
+; GFX1250-FAKE16: ; %bb.0:
+; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-FAKE16-NEXT: v_nop
+; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-FAKE16-NEXT: s_load_u16 s0, s[0:1], 0x8 nv
+; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-FAKE16-NEXT: global_store_b16 v0, v1, s[2:3]
+; GFX1250-FAKE16-NEXT: s_endpgm
+;
+; GFX1250-REAL16-LABEL: incorrect_type_i16_block_count_x:
+; GFX1250-REAL16: ; %bb.0:
+; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-REAL16-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-REAL16-NEXT: v_nop
+; GFX1250-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-REAL16-NEXT: s_load_u16 s0, s[0:1], 0x8 nv
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
+; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, s0
+; GFX1250-REAL16-NEXT: global_store_b16 v1, v0, s[2:3]
+; GFX1250-REAL16-NEXT: s_endpgm
%imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
%load = load i16, ptr addrspace(4) %imp_arg_ptr
store i16 %load, ptr addrspace(1) %out
@@ -1172,6 +1185,3 @@ define amdgpu_kernel void @preload_block_count_z_workgroup_size_z_remainder_z(pt
}
attributes #0 = { "amdgpu-agpr-alloc"="0" "amdgpu-no-completion-action" "amdgpu-no-default-queue" "amdgpu-no-dispatch-id" "amdgpu-no-dispatch-ptr" "amdgpu-no-heap-ptr" "amdgpu-no-hostcall-ptr" "amdgpu-no-lds-kernel-id" "amdgpu-no-multigrid-sync-arg" "amdgpu-no-queue-ptr" "amdgpu-no-workgroup-id-x" "amdgpu-no-workgroup-id-y" "amdgpu-no-workgroup-id-z" "amdgpu-no-workitem-id-x" "amdgpu-no-workitem-id-y" "amdgpu-no-workitem-id-z" }
-;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
-; GFX1250-FAKE16: {{.*}}
-; GFX1250-REAL16: {{.*}}
>From a02211355982741dee69866f284795223231faf0 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Wed, 23 Sep 2026 05:19:43 -0500
Subject: [PATCH 2/6] Use legalization approach
Signed-off-by: John Lu <John.Lu at amd.com>
---
llvm/include/llvm/CodeGen/TargetLowering.h | 6 +
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 +-
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 42 +++
llvm/lib/Target/AMDGPU/SIISelLowering.h | 1 +
llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll | 4 +-
llvm/test/CodeGen/AMDGPU/fcmp.f16.ll | 88 ++---
llvm/test/CodeGen/AMDGPU/fmax3.ll | 48 +--
llvm/test/CodeGen/AMDGPU/fmaximum.ll | 6 +-
llvm/test/CodeGen/AMDGPU/fmin3.ll | 48 +--
llvm/test/CodeGen/AMDGPU/fminimum.ll | 6 +-
llvm/test/CodeGen/AMDGPU/frem.ll | 351 +++++++++---------
llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 275 ++++++--------
llvm/test/CodeGen/AMDGPU/load-constant-i16.ll | 158 +++-----
llvm/test/CodeGen/AMDGPU/load-constant-i8.ll | 65 ++--
llvm/test/CodeGen/AMDGPU/min.ll | 11 +-
...-to-valu-pseudo-scalar-trans-f16-true16.ll | 50 +--
.../AMDGPU/preload-implicit-kernargs.ll | 39 +-
23 files changed, 574 insertions(+), 639 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 18255b429d995..9d909c100ab19 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4839,6 +4839,12 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
return isTypeLegal(VT);
}
+ /// Overload that provides context about the specific node being optimized.
+ /// By default, calls the version without the node parameter.
+ virtual bool isTypeDesirableForOp(unsigned Opc, EVT VT, SDNode *N) const {
+ return isTypeDesirableForOp(Opc, VT);
+ }
+
/// Return true if it is profitable for dag combiner to transform a floating
/// point op of specified opcode to a equivalent op of an integer
/// type. e.g. f32 load -> i32 load can be profitable on ARM.
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index f94c861200f20..c43f6990fb18c 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -17970,7 +17970,8 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
// fold (truncate (load x)) -> (smaller load x)
// fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
- if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getOpcode(), VT)) {
+ if (!LegalTypes ||
+ TLI.isTypeDesirableForOp(N0.getOpcode(), VT, N0.getNode())) {
if (SDValue Reduced = reduceLoadWidth(N))
return Reduced;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 40fcc2e25525a..4fef4d04de7f6 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -2505,6 +2505,24 @@ bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
return TargetLowering::isTypeDesirableForOp(Op, VT);
}
+bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT,
+ SDNode *N) const {
+ // Do not convert uniform i32 loads to 16-bit.
+ // Uniform 16-bit loads are legalized to i16 = trunc (zextload i16->i32)
+ // to match subword load patterns. Allowing conversion back to a 16-bit
+ // load would create an infinite loop.
+ if (Subtarget->hasScalarSubwordLoads() && Op == ISD::LOAD && !VT.isVector() &&
+ VT.getSizeInBits() == 16) {
+ auto *Load = dyn_cast<LoadSDNode>(N);
+ if (Load && Load->getValueType(0) == MVT::i32 && !Load->isDivergent() &&
+ AMDGPU::isUniformMMO(Load->getMemOperand())) {
+ return false;
+ }
+ }
+
+ return isTypeDesirableForOp(Op, VT);
+}
+
MachinePointerInfo
SITargetLowering::getKernargSegmentPtrInfo(MachineFunction &MF) const {
// This isn't really a constant pool but close enough.
@@ -13605,6 +13623,30 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
MachineMemOperand *MMO = Load->getMemOperand();
if (ExtType == ISD::NON_EXTLOAD && MemVT.getSizeInBits() < 32) {
+ // Legalize uniform 16-bit loads to i16 = trunc (zextload i16->i32)
+ // to match subword load patterns.
+ if (!MemVT.isVector() && MemVT.getSizeInBits() == 16 &&
+ isTypeLegal(MemVT) && Subtarget->hasScalarSubwordLoads() &&
+ !Load->isDivergent() && AMDGPU::isUniformMMO(MMO)) {
+ SDValue Chain = Load->getChain();
+ SDValue BasePtr = Load->getBasePtr();
+
+ // Load as i16 and zero-extend to i32 (matches S_LOAD_U16 behavior)
+ SDValue NewLD = DAG.getExtLoad(ISD::ZEXTLOAD, DL, MVT::i32, Chain,
+ BasePtr, MVT::i16, MMO);
+
+ // Truncate back to i16
+ SDValue Trunc = DAG.getNode(ISD::TRUNCATE, DL, MVT::i16, NewLD);
+
+ // For f16/bf16, bitcast from i16 to the original fp type
+ SDValue Result = (MemVT == MVT::i16)
+ ? Trunc
+ : DAG.getNode(ISD::BITCAST, DL, MemVT, Trunc);
+
+ SDValue Ops[] = {Result, NewLD.getValue(1)};
+ return DAG.getMergeValues(Ops, DL);
+ }
+
if (MemVT == MVT::i16 && isTypeLegal(MVT::i16))
return SDValue();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 2ce36586402b6..8fdc82db000a0 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -417,6 +417,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override;
bool isTypeDesirableForOp(unsigned Op, EVT VT) const override;
+ bool isTypeDesirableForOp(unsigned Op, EVT VT, SDNode *N) const override;
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
diff --git a/llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll b/llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll
index 7fe521102be99..25a488f2b795f 100644
--- a/llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll
+++ b/llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll
@@ -126,9 +126,9 @@ define amdgpu_kernel void @d16_two_order_group(ptr addrspace(3) %in1, ptr %in2)
; GFX12-NEXT: flat_load_d16_b16 v0, v[1:2]
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v3, s0
-; GFX12-NEXT: ds_load_u16_d16_hi v0, v3
+; GFX12-NEXT: ds_load_u16 v3, v3
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_or_b16 v0.l, v0.h, v0.l
+; GFX12-NEXT: v_or_b16 v0.l, v3.l, v0.l
; GFX12-NEXT: flat_store_b16 v[1:2], v0
; GFX12-NEXT: s_endpgm
%i16a = load i16, ptr addrspace(3) %in1, align 2
diff --git a/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll b/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
index 1720f242fcc96..3685fb0d5d18f 100644
--- a/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
@@ -131,13 +131,13 @@ define amdgpu_kernel void @fcmp_f16_lt(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -303,9 +303,9 @@ define amdgpu_kernel void @fcmp_f16_lt_abs(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
@@ -483,13 +483,13 @@ define amdgpu_kernel void @fcmp_f16_eq(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_eq_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -655,13 +655,13 @@ define amdgpu_kernel void @fcmp_f16_le(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -827,13 +827,13 @@ define amdgpu_kernel void @fcmp_f16_gt(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -999,13 +999,13 @@ define amdgpu_kernel void @fcmp_f16_lg(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1171,13 +1171,13 @@ define amdgpu_kernel void @fcmp_f16_ge(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_ge_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_ge_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1343,13 +1343,13 @@ define amdgpu_kernel void @fcmp_f16_o(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1515,13 +1515,13 @@ define amdgpu_kernel void @fcmp_f16_u(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1687,13 +1687,13 @@ define amdgpu_kernel void @fcmp_f16_nge(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1859,13 +1859,13 @@ define amdgpu_kernel void @fcmp_f16_nlg(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_nlg_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_nlg_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -2031,13 +2031,13 @@ define amdgpu_kernel void @fcmp_f16_ngt(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_ngt_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_ngt_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -2203,13 +2203,13 @@ define amdgpu_kernel void @fcmp_f16_nle(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -2375,13 +2375,13 @@ define amdgpu_kernel void @fcmp_f16_neq(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_neq_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_neq_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -2547,13 +2547,13 @@ define amdgpu_kernel void @fcmp_f16_nlt(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_nlt_f16_e32 vcc_lo, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_cmp_nlt_f16_e32 vcc_lo, v0.l, v1.l
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fmax3.ll b/llvm/test/CodeGen/AMDGPU/fmax3.ll
index f34efb4f4af2b..6618966a8e1ce 100644
--- a/llvm/test/CodeGen/AMDGPU/fmax3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmax3.ll
@@ -696,15 +696,15 @@ define amdgpu_kernel void @test_fmax3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
; GFX12-TRUE16-NEXT: s_mov_b32 s17, s5
; GFX12-TRUE16-NEXT: s_mov_b32 s20, s6
; GFX12-TRUE16-NEXT: s_mov_b32 s21, s7
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v2, off, s[16:19], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[20:23], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_max3_num_f16 v0.l, v0.l, v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_max3_num_f16 v0.l, v1.l, v2.l, v0.l
; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -760,15 +760,15 @@ define amdgpu_kernel void @test_fmax3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
; GFX1250-TRUE16-NEXT: s_mov_b32 s17, s13
; GFX1250-TRUE16-NEXT: s_mov_b32 s20, s14
; GFX1250-TRUE16-NEXT: s_mov_b32 s21, s15
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v2, off, s[16:19], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v0, off, s[20:23], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s0, s8
; GFX1250-TRUE16-NEXT: s_mov_b32 s1, s9
-; GFX1250-TRUE16-NEXT: v_max3_num_f16 v0.l, v0.l, v0.h, v1.l
+; GFX1250-TRUE16-NEXT: v_max3_num_f16 v0.l, v1.l, v2.l, v0.l
; GFX1250-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-TRUE16-NEXT: s_endpgm
;
@@ -1044,24 +1044,24 @@ define amdgpu_kernel void @test_fmax3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
; GFX12-TRUE16-NEXT: s_mov_b32 s15, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s18, s10
; GFX12-TRUE16-NEXT: s_mov_b32 s19, s11
-; GFX12-TRUE16-NEXT: s_mov_b32 s22, s10
-; GFX12-TRUE16-NEXT: s_mov_b32 s23, s11
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s16, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s17, s5
-; GFX12-TRUE16-NEXT: s_mov_b32 s20, s6
-; GFX12-TRUE16-NEXT: s_mov_b32 s21, s7
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s11
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v2, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_max3_num_f16 v0.l, v1.l, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_max3_num_f16 v0.l, v2.l, v1.l, v0.l
; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -1108,24 +1108,24 @@ define amdgpu_kernel void @test_fmax3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
; GFX1250-TRUE16-NEXT: s_mov_b32 s7, s3
; GFX1250-TRUE16-NEXT: s_mov_b32 s18, s2
; GFX1250-TRUE16-NEXT: s_mov_b32 s19, s3
-; GFX1250-TRUE16-NEXT: s_mov_b32 s22, s2
-; GFX1250-TRUE16-NEXT: s_mov_b32 s23, s3
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s4, s10
; GFX1250-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1250-TRUE16-NEXT: s_mov_b32 s16, s12
; GFX1250-TRUE16-NEXT: s_mov_b32 s17, s13
-; GFX1250-TRUE16-NEXT: s_mov_b32 s20, s14
-; GFX1250-TRUE16-NEXT: s_mov_b32 s21, s15
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: s_mov_b32 s12, s14
+; GFX1250-TRUE16-NEXT: s_mov_b32 s13, s15
+; GFX1250-TRUE16-NEXT: s_mov_b32 s14, s2
+; GFX1250-TRUE16-NEXT: s_mov_b32 s15, s3
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v2, off, s[12:15], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s0, s8
; GFX1250-TRUE16-NEXT: s_mov_b32 s1, s9
-; GFX1250-TRUE16-NEXT: v_max3_num_f16 v0.l, v1.l, v0.l, v0.h
+; GFX1250-TRUE16-NEXT: v_max3_num_f16 v0.l, v2.l, v1.l, v0.l
; GFX1250-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
index 213d6ffeb9f00..dffa28a5c52fd 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -1492,11 +1492,11 @@ define amdgpu_kernel void @fmaximum_f16_move_to_valu(ptr addrspace(1) %out, ptr
; GFX12-SDAG-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-SDAG-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3] scope:SCOPE_SYS
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: global_load_d16_hi_b16 v0, v1, s[4:5] scope:SCOPE_SYS
+; GFX12-SDAG-TRUE16-NEXT: global_load_u16 v2, v1, s[4:5] scope:SCOPE_SYS
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fmin3.ll b/llvm/test/CodeGen/AMDGPU/fmin3.ll
index c03e81e6b7129..bcd50ee8f47fd 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin3.ll
@@ -696,15 +696,15 @@ define amdgpu_kernel void @test_fmin3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
; GFX12-TRUE16-NEXT: s_mov_b32 s17, s5
; GFX12-TRUE16-NEXT: s_mov_b32 s20, s6
; GFX12-TRUE16-NEXT: s_mov_b32 s21, s7
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v2, off, s[16:19], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[20:23], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_min3_num_f16 v0.l, v0.l, v0.h, v1.l
+; GFX12-TRUE16-NEXT: v_min3_num_f16 v0.l, v1.l, v2.l, v0.l
; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -760,15 +760,15 @@ define amdgpu_kernel void @test_fmin3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
; GFX1250-TRUE16-NEXT: s_mov_b32 s17, s13
; GFX1250-TRUE16-NEXT: s_mov_b32 s20, s14
; GFX1250-TRUE16-NEXT: s_mov_b32 s21, s15
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v2, off, s[16:19], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v0, off, s[20:23], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s0, s8
; GFX1250-TRUE16-NEXT: s_mov_b32 s1, s9
-; GFX1250-TRUE16-NEXT: v_min3_num_f16 v0.l, v0.l, v0.h, v1.l
+; GFX1250-TRUE16-NEXT: v_min3_num_f16 v0.l, v1.l, v2.l, v0.l
; GFX1250-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-TRUE16-NEXT: s_endpgm
;
@@ -1044,24 +1044,24 @@ define amdgpu_kernel void @test_fmin3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
; GFX12-TRUE16-NEXT: s_mov_b32 s15, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s18, s10
; GFX12-TRUE16-NEXT: s_mov_b32 s19, s11
-; GFX12-TRUE16-NEXT: s_mov_b32 s22, s10
-; GFX12-TRUE16-NEXT: s_mov_b32 s23, s11
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s16, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s17, s5
-; GFX12-TRUE16-NEXT: s_mov_b32 s20, s6
-; GFX12-TRUE16-NEXT: s_mov_b32 s21, s7
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
+; GFX12-TRUE16-NEXT: s_mov_b32 s6, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s7, s11
+; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_u16 v2, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_min3_num_f16 v0.l, v1.l, v0.l, v0.h
+; GFX12-TRUE16-NEXT: v_min3_num_f16 v0.l, v2.l, v1.l, v0.l
; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -1108,24 +1108,24 @@ define amdgpu_kernel void @test_fmin3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
; GFX1250-TRUE16-NEXT: s_mov_b32 s7, s3
; GFX1250-TRUE16-NEXT: s_mov_b32 s18, s2
; GFX1250-TRUE16-NEXT: s_mov_b32 s19, s3
-; GFX1250-TRUE16-NEXT: s_mov_b32 s22, s2
-; GFX1250-TRUE16-NEXT: s_mov_b32 s23, s3
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s4, s10
; GFX1250-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1250-TRUE16-NEXT: s_mov_b32 s16, s12
; GFX1250-TRUE16-NEXT: s_mov_b32 s17, s13
-; GFX1250-TRUE16-NEXT: s_mov_b32 s20, s14
-; GFX1250-TRUE16-NEXT: s_mov_b32 s21, s15
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: s_mov_b32 s12, s14
+; GFX1250-TRUE16-NEXT: s_mov_b32 s13, s15
+; GFX1250-TRUE16-NEXT: s_mov_b32 s14, s2
+; GFX1250-TRUE16-NEXT: s_mov_b32 s15, s3
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_u16 v2, off, s[12:15], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s0, s8
; GFX1250-TRUE16-NEXT: s_mov_b32 s1, s9
-; GFX1250-TRUE16-NEXT: v_min3_num_f16 v0.l, v1.l, v0.l, v0.h
+; GFX1250-TRUE16-NEXT: v_min3_num_f16 v0.l, v2.l, v1.l, v0.l
; GFX1250-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
index 094dd76e0f945..11acac09aa30f 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -1492,11 +1492,11 @@ define amdgpu_kernel void @fminimum_f16_move_to_valu(ptr addrspace(1) %out, ptr
; GFX12-SDAG-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-SDAG-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3] scope:SCOPE_SYS
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: global_load_d16_hi_b16 v0, v1, s[4:5] scope:SCOPE_SYS
+; GFX12-SDAG-TRUE16-NEXT: global_load_u16 v2, v1, s[4:5] scope:SCOPE_SYS
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v0.h
+; GFX12-SDAG-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index f202227662c40..e2ea47c6b27fb 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -978,244 +978,249 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX1200-TRUE16-LABEL: frem_f16:
; GFX1200-TRUE16: ; %bb.0:
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1200-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1200-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
+; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v0, 0
; GFX1200-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[10:11]
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v1, v1, s[0:1] offset:8
+; GFX1200-TRUE16-NEXT: global_load_u16 v1, v0, s[2:3]
+; GFX1200-TRUE16-NEXT: global_load_u16 v0, v0, s[4:5] offset:8
; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x1
-; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s6, v1
; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1200-TRUE16-NEXT: s_and_b32 s0, s0, 0x7fff
-; GFX1200-TRUE16-NEXT: s_and_b32 s2, s1, 0x7fff
-; GFX1200-TRUE16-NEXT: s_cvt_f32_f16 s1, s0
-; GFX1200-TRUE16-NEXT: s_cvt_f32_f16 s0, s2
+; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1200-TRUE16-NEXT: s_and_b32 s2, s6, 0x7fff
+; GFX1200-TRUE16-NEXT: s_and_b32 s4, s3, 0x7fff
+; GFX1200-TRUE16-NEXT: s_cvt_f32_f16 s5, s2
+; GFX1200-TRUE16-NEXT: s_cvt_f32_f16 s4, s4
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1200-TRUE16-NEXT: s_cmp_ngt_f32 s1, s0
+; GFX1200-TRUE16-NEXT: s_cmp_ngt_f32 s5, s4
; GFX1200-TRUE16-NEXT: s_cbranch_scc0 .LBB0_2
; GFX1200-TRUE16-NEXT: ; %bb.1: ; %frem.else
-; GFX1200-TRUE16-NEXT: s_cmp_eq_f32 s1, s0
-; GFX1200-TRUE16-NEXT: v_and_b16 v0.h, 0x8000, v0.l
-; GFX1200-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1200-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.l, v0.h, s2
-; GFX1200-TRUE16-NEXT: s_mov_b32 s2, 0
+; GFX1200-TRUE16-NEXT: s_cmp_eq_f32 s5, s4
+; GFX1200-TRUE16-NEXT: v_and_b16 v0.l, 0x8000, s6
+; GFX1200-TRUE16-NEXT: s_cselect_b32 s7, -1, 0
+; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT: v_cndmask_b16 v0.l, s6, v0.l, s7
+; GFX1200-TRUE16-NEXT: s_mov_b32 s6, 0
; GFX1200-TRUE16-NEXT: s_branch .LBB0_3
; GFX1200-TRUE16-NEXT: .LBB0_2:
-; GFX1200-TRUE16-NEXT: s_mov_b32 s2, -1
-; GFX1200-TRUE16-NEXT: ; implicit-def: $vgpr2
+; GFX1200-TRUE16-NEXT: s_mov_b32 s6, -1
+; GFX1200-TRUE16-NEXT: ; implicit-def: $vgpr0
; GFX1200-TRUE16-NEXT: .LBB0_3: ; %Flow18
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1200-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1200-TRUE16-NEXT: s_and_b32 s6, s6, exec_lo
+; GFX1200-TRUE16-NEXT: s_cselect_b32 s6, 1, 0
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX1200-TRUE16-NEXT: s_cmp_lg_u32 s6, 1
; GFX1200-TRUE16-NEXT: s_cbranch_scc1 .LBB0_8
; GFX1200-TRUE16-NEXT: ; %bb.4: ; %frem.compute
-; GFX1200-TRUE16-NEXT: v_frexp_mant_f32_e32 v3, s0
-; GFX1200-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v5, s0
-; GFX1200-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v4, s1
-; GFX1200-TRUE16-NEXT: v_frexp_mant_f32_e32 v2, s1
+; GFX1200-TRUE16-NEXT: v_frexp_mant_f32_e32 v1, s4
+; GFX1200-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v3, s4
+; GFX1200-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v2, s5
+; GFX1200-TRUE16-NEXT: v_frexp_mant_f32_e32 v0, s5
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-TRUE16-NEXT: v_ldexp_f32 v3, v3, 1
-; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s0, v5
-; GFX1200-TRUE16-NEXT: v_add_nc_u32_e32 v5, -1, v5
-; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s1, v4
-; GFX1200-TRUE16-NEXT: v_ldexp_f32 v2, v2, 11
-; GFX1200-TRUE16-NEXT: v_div_scale_f32 v6, null, v3, v3, 1.0
+; GFX1200-TRUE16-NEXT: v_ldexp_f32 v1, v1, 1
+; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s4, v3
+; GFX1200-TRUE16-NEXT: v_add_nc_u32_e32 v3, -1, v3
+; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
+; GFX1200-TRUE16-NEXT: v_ldexp_f32 v0, v0, 11
+; GFX1200-TRUE16-NEXT: v_div_scale_f32 v4, null, v1, v1, 1.0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-TRUE16-NEXT: v_not_b32_e32 v5, v5
-; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v7, v6
+; GFX1200-TRUE16-NEXT: v_not_b32_e32 v3, v3
+; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v5, v4
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(TRANS32_DEP_1)
-; GFX1200-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v4
-; GFX1200-TRUE16-NEXT: v_div_scale_f32 v4, vcc_lo, 1.0, v3, 1.0
+; GFX1200-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v2
+; GFX1200-TRUE16-NEXT: v_div_scale_f32 v2, vcc_lo, 1.0, v1, 1.0
; GFX1200-TRUE16-NEXT: s_denorm_mode 15
-; GFX1200-TRUE16-NEXT: v_fma_f32 v8, -v6, v7, 1.0
+; GFX1200-TRUE16-NEXT: v_fma_f32 v6, -v4, v5, 1.0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v7, v8, v7
-; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v8, v4, v7
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v5, v6, v5
+; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v6, v2, v5
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_f32 v9, -v6, v8, v4
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v8, v9, v7
+; GFX1200-TRUE16-NEXT: v_fma_f32 v7, -v4, v6, v2
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v6, v7, v5
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_f32 v4, -v6, v8, v4
+; GFX1200-TRUE16-NEXT: v_fma_f32 v2, -v4, v6, v2
; GFX1200-TRUE16-NEXT: s_denorm_mode 12
-; GFX1200-TRUE16-NEXT: v_div_fmas_f32 v4, v4, v7, v8
-; GFX1200-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v5
+; GFX1200-TRUE16-NEXT: v_div_fmas_f32 v2, v2, v5, v6
+; GFX1200-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v3
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-TRUE16-NEXT: v_div_fixup_f32 v4, v4, v3, 1.0
+; GFX1200-TRUE16-NEXT: v_div_fixup_f32 v2, v2, v1, 1.0
; GFX1200-TRUE16-NEXT: s_cbranch_vccnz .LBB0_7
; GFX1200-TRUE16-NEXT: ; %bb.5: ; %frem.loop_body.preheader
-; GFX1200-TRUE16-NEXT: s_sub_co_i32 s0, s1, s0
+; GFX1200-TRUE16-NEXT: s_sub_co_i32 s4, s5, s4
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_add_co_i32 s0, s0, 11
+; GFX1200-TRUE16-NEXT: s_add_co_i32 s4, s4, 11
; GFX1200-TRUE16-NEXT: .LBB0_6: ; %frem.loop_body
; GFX1200-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v5, v2, v4
+; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v3, v0, v2
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_add_co_i32 s0, s0, -11
+; GFX1200-TRUE16-NEXT: s_add_co_i32 s4, s4, -11
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_cmp_gt_i32 s0, 11
-; GFX1200-TRUE16-NEXT: v_rndne_f32_e32 v5, v5
+; GFX1200-TRUE16-NEXT: s_cmp_gt_i32 s4, 11
+; GFX1200-TRUE16-NEXT: v_rndne_f32_e32 v3, v3
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v2, v5, v3
+; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v0, v3, v1
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v2
-; GFX1200-TRUE16-NEXT: v_add_f32_e32 v5, v2, v3
+; GFX1200-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v0
+; GFX1200-TRUE16-NEXT: v_add_f32_e32 v3, v0, v1
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
+; GFX1200-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_ldexp_f32 v2, v2, 11
+; GFX1200-TRUE16-NEXT: v_ldexp_f32 v0, v0, 11
; GFX1200-TRUE16-NEXT: s_cbranch_scc1 .LBB0_6
; GFX1200-TRUE16-NEXT: .LBB0_7: ; %frem.loop_exit
-; GFX1200-TRUE16-NEXT: ; implicit-def: $vgpr2
+; GFX1200-TRUE16-NEXT: ; implicit-def: $vgpr0
; GFX1200-TRUE16-NEXT: .LBB0_8: ; %Flow19
-; GFX1200-TRUE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x7fff, v0
-; GFX1200-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0, v1.l
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-TRUE16-NEXT: v_cmp_nle_f16_e64 s0, 0x7c00, v0.l
-; GFX1200-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
-; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v2.l, s0
-; GFX1200-TRUE16-NEXT: global_store_b16 v3, v0, s[8:9]
+; GFX1200-TRUE16-NEXT: s_cmp_lg_f16 s3, 0
+; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1200-TRUE16-NEXT: s_cselect_b32 s3, -1, 0
+; GFX1200-TRUE16-NEXT: s_cmp_nge_f16 s2, 0x7c00
+; GFX1200-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
+; GFX1200-TRUE16-NEXT: s_and_b32 s2, s2, s3
+; GFX1200-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v0.l, s2
+; GFX1200-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1200-TRUE16-NEXT: s_endpgm
;
; GFX1200-FAKE16-LABEL: frem_f16:
; GFX1200-FAKE16: ; %bb.0:
; GFX1200-FAKE16-NEXT: s_clause 0x1
-; GFX1200-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
-; GFX1200-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1200-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX1200-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX1200-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-FAKE16-NEXT: s_clause 0x1
-; GFX1200-FAKE16-NEXT: global_load_u16 v0, v1, s[10:11]
-; GFX1200-FAKE16-NEXT: global_load_u16 v1, v1, s[0:1] offset:8
+; GFX1200-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
+; GFX1200-FAKE16-NEXT: global_load_u16 v0, v0, s[6:7] offset:8
; GFX1200-FAKE16-NEXT: s_wait_loadcnt 0x1
-; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
+; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
; GFX1200-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
-; GFX1200-FAKE16-NEXT: s_and_b32 s0, s0, 0x7fff
-; GFX1200-FAKE16-NEXT: s_and_b32 s2, s1, 0x7fff
-; GFX1200-FAKE16-NEXT: s_cvt_f32_f16 s1, s0
-; GFX1200-FAKE16-NEXT: s_cvt_f32_f16 s0, s2
+; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s3, v0
+; GFX1200-FAKE16-NEXT: s_and_b32 s2, s4, 0x7fff
+; GFX1200-FAKE16-NEXT: s_and_b32 s5, s3, 0x7fff
+; GFX1200-FAKE16-NEXT: s_cvt_f32_f16 s6, s2
+; GFX1200-FAKE16-NEXT: s_cvt_f32_f16 s5, s5
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1200-FAKE16-NEXT: s_cmp_ngt_f32 s1, s0
+; GFX1200-FAKE16-NEXT: s_cmp_ngt_f32 s6, s5
; GFX1200-FAKE16-NEXT: s_cbranch_scc0 .LBB0_2
; GFX1200-FAKE16-NEXT: ; %bb.1: ; %frem.else
-; GFX1200-FAKE16-NEXT: s_cmp_eq_f32 s1, s0
-; GFX1200-FAKE16-NEXT: v_and_b32_e32 v2, 0x8000, v0
-; GFX1200-FAKE16-NEXT: s_mov_b32 s2, 0
+; GFX1200-FAKE16-NEXT: s_and_b32 s7, 0x8000, s4
+; GFX1200-FAKE16-NEXT: s_cmp_eq_f32 s6, s5
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v0, s7
+; GFX1200-FAKE16-NEXT: s_mov_b32 s7, 0
; GFX1200-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc_lo
+; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v0, s4, v0, vcc_lo
; GFX1200-FAKE16-NEXT: s_branch .LBB0_3
; GFX1200-FAKE16-NEXT: .LBB0_2:
-; GFX1200-FAKE16-NEXT: s_mov_b32 s2, -1
-; GFX1200-FAKE16-NEXT: ; implicit-def: $vgpr2
+; GFX1200-FAKE16-NEXT: s_mov_b32 s7, -1
+; GFX1200-FAKE16-NEXT: ; implicit-def: $vgpr0
; GFX1200-FAKE16-NEXT: .LBB0_3: ; %Flow18
-; GFX1200-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
-; GFX1200-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
-; GFX1200-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
-; GFX1200-FAKE16-NEXT: s_cmp_lg_u32 s2, 1
+; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-FAKE16-NEXT: s_and_b32 s7, s7, exec_lo
+; GFX1200-FAKE16-NEXT: s_cselect_b32 s7, 1, 0
+; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-FAKE16-NEXT: s_cmp_lg_u32 s7, 1
; GFX1200-FAKE16-NEXT: s_cbranch_scc1 .LBB0_9
; GFX1200-FAKE16-NEXT: ; %bb.4: ; %frem.compute
-; GFX1200-FAKE16-NEXT: v_frexp_mant_f32_e32 v3, s0
-; GFX1200-FAKE16-NEXT: v_frexp_mant_f32_e32 v2, s1
-; GFX1200-FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v5, s1
+; GFX1200-FAKE16-NEXT: v_frexp_mant_f32_e32 v1, s5
+; GFX1200-FAKE16-NEXT: v_frexp_mant_f32_e32 v0, s6
+; GFX1200-FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v3, s6
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v3, v3, 1
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v4, v2, 11
-; GFX1200-FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v2, s0
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v1, v1, 1
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v2, v0, 11
+; GFX1200-FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v0, s5
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s1, v5
-; GFX1200-FAKE16-NEXT: v_div_scale_f32 v7, null, v3, v3, 1.0
+; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
+; GFX1200-FAKE16-NEXT: v_div_scale_f32 v5, null, v1, v1, 1.0
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s0, v2
-; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v2, -1, v2
-; GFX1200-FAKE16-NEXT: v_rcp_f32_e32 v8, v7
+; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s5, v0
+; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v0, -1, v0
+; GFX1200-FAKE16-NEXT: v_rcp_f32_e32 v6, v5
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_not_b32_e32 v6, v2
-; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v6, v6, v5
-; GFX1200-FAKE16-NEXT: v_div_scale_f32 v5, vcc_lo, 1.0, v3, 1.0
+; GFX1200-FAKE16-NEXT: v_not_b32_e32 v4, v0
+; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v4, v4, v3
+; GFX1200-FAKE16-NEXT: v_div_scale_f32 v3, vcc_lo, 1.0, v1, 1.0
; GFX1200-FAKE16-NEXT: s_denorm_mode 15
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_fma_f32 v9, -v7, v8, 1.0
-; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v8, v9, v8
+; GFX1200-FAKE16-NEXT: v_fma_f32 v7, -v5, v6, 1.0
+; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v6, v7, v6
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v9, v5, v8
-; GFX1200-FAKE16-NEXT: v_fma_f32 v10, -v7, v9, v5
+; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v7, v3, v6
+; GFX1200-FAKE16-NEXT: v_fma_f32 v8, -v5, v7, v3
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v9, v10, v8
-; GFX1200-FAKE16-NEXT: v_fma_f32 v5, -v7, v9, v5
+; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v7, v8, v6
+; GFX1200-FAKE16-NEXT: v_fma_f32 v3, -v5, v7, v3
; GFX1200-FAKE16-NEXT: s_denorm_mode 12
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1200-FAKE16-NEXT: v_div_fmas_f32 v5, v5, v8, v9
-; GFX1200-FAKE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v6
-; GFX1200-FAKE16-NEXT: v_div_fixup_f32 v5, v5, v3, 1.0
+; GFX1200-FAKE16-NEXT: v_div_fmas_f32 v3, v3, v6, v7
+; GFX1200-FAKE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v4
+; GFX1200-FAKE16-NEXT: v_div_fixup_f32 v3, v3, v1, 1.0
; GFX1200-FAKE16-NEXT: s_cbranch_vccnz .LBB0_8
; GFX1200-FAKE16-NEXT: ; %bb.5: ; %frem.loop_body.preheader
-; GFX1200-FAKE16-NEXT: s_sub_co_i32 s0, s1, s0
+; GFX1200-FAKE16-NEXT: s_sub_co_i32 s5, s6, s5
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: s_add_co_i32 s0, s0, 11
+; GFX1200-FAKE16-NEXT: s_add_co_i32 s5, s5, 11
; GFX1200-FAKE16-NEXT: .LBB0_6: ; %frem.loop_body
; GFX1200-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v7, v4
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v5, v2
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: s_add_co_i32 s0, s0, -11
+; GFX1200-FAKE16-NEXT: s_add_co_i32 s5, s5, -11
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: s_cmp_gt_i32 s0, 11
-; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v4, v7, v5
+; GFX1200-FAKE16-NEXT: s_cmp_gt_i32 s5, 11
+; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v2, v5, v3
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_rndne_f32_e32 v4, v4
-; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v4, 0x80000000, v4
+; GFX1200-FAKE16-NEXT: v_rndne_f32_e32 v2, v2
+; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_fma_f32 v4, v4, v3, v7
-; GFX1200-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v4
-; GFX1200-FAKE16-NEXT: v_add_f32_e32 v6, v4, v3
+; GFX1200-FAKE16-NEXT: v_fma_f32 v2, v2, v1, v5
+; GFX1200-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v2
+; GFX1200-FAKE16-NEXT: v_add_f32_e32 v4, v2, v1
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v4, v4, 11
+; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v2, v2, 11
; GFX1200-FAKE16-NEXT: s_cbranch_scc1 .LBB0_6
; GFX1200-FAKE16-NEXT: ; %bb.7: ; %Flow
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v6, s0
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v4, v7
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v4, s5
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v2, v5
; GFX1200-FAKE16-NEXT: .LBB0_8: ; %frem.loop_exit
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v6, -10, v6
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v4, v4, v6
+; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v4, -10, v4
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v2, v2, v4
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v5, v4, v5
-; GFX1200-FAKE16-NEXT: v_rndne_f32_e32 v5, v5
+; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v3, v2, v3
+; GFX1200-FAKE16-NEXT: v_rndne_f32_e32 v3, v3
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
-; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v4, v5, v3
+; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
+; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v2, v3, v1
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v4
-; GFX1200-FAKE16-NEXT: v_add_f32_e32 v3, v4, v3
+; GFX1200-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v2
+; GFX1200-FAKE16-NEXT: v_add_f32_e32 v1, v2, v1
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo
+; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc_lo
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v2, v3, v2
-; GFX1200-FAKE16-NEXT: v_cvt_f16_f32_e32 v2, v2
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v0, v1, v0
+; GFX1200-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_bfi_b32 v2, 0x7fff, v2, v0
+; GFX1200-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, s4
; GFX1200-FAKE16-NEXT: .LBB0_9: ; %Flow19
-; GFX1200-FAKE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x7fff, v0
-; GFX1200-FAKE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0, v1
-; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-FAKE16-NEXT: v_cmp_nle_f16_e64 s0, 0x7c00, v0
-; GFX1200-FAKE16-NEXT: s_and_b32 vcc_lo, s0, vcc_lo
+; GFX1200-FAKE16-NEXT: s_cmp_lg_f16 s3, 0
+; GFX1200-FAKE16-NEXT: s_cselect_b32 s3, -1, 0
+; GFX1200-FAKE16-NEXT: s_cmp_nge_f16 s2, 0x7c00
+; GFX1200-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1200-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX1200-FAKE16-NEXT: s_and_b32 vcc_lo, s2, s3
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
-; GFX1200-FAKE16-NEXT: global_store_b16 v3, v0, s[8:9]
+; GFX1200-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_cndmask_b32 v0, 0x7e00, v0
+; GFX1200-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX1200-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %in2) #0 {
%gep2 = getelementptr half, ptr addrspace(1) %in2, i32 4
@@ -1636,34 +1641,35 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1200-TRUE16-NEXT: s_clause 0x1
; GFX1200-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1200-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v2, 0
; GFX1200-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v3, v1, s[4:5] offset:8
+; GFX1200-TRUE16-NEXT: global_load_u16 v3, v2, s[2:3]
+; GFX1200-TRUE16-NEXT: global_load_u16 v4, v2, s[4:5] offset:8
; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v3.l
+; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v4.l
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v0, v0
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v4, v2, v0, neg(0) op_sel_hi:[1,0,0]
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v1, v3, v0, neg(0) op_sel_hi:[1,0,0]
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v4, v5, v0
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v4, v1, v3 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v1, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v4, v1, v3 op_sel_hi:[1,0,1]
; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1200-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
+; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v4.l, v1.l
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v3.l
-; GFX1200-TRUE16-NEXT: global_store_b16 v1, v2, s[0:1]
+; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v3.l, v0.l, v4.l
+; GFX1200-TRUE16-NEXT: global_store_b16 v2, v3, s[0:1]
; GFX1200-TRUE16-NEXT: s_endpgm
;
; GFX1200-FAKE16-LABEL: fast_frem_f16:
@@ -2119,34 +2125,35 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX1200-TRUE16-NEXT: s_clause 0x1
; GFX1200-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1200-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v2, 0
; GFX1200-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
-; GFX1200-TRUE16-NEXT: global_load_d16_b16 v3, v1, s[4:5] offset:8
+; GFX1200-TRUE16-NEXT: global_load_u16 v3, v2, s[2:3]
+; GFX1200-TRUE16-NEXT: global_load_u16 v4, v2, s[4:5] offset:8
; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v3.l
+; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v4.l
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v0, v0
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v4, v2, v0, neg(0) op_sel_hi:[1,0,0]
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v1, v3, v0, neg(0) op_sel_hi:[1,0,0]
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v4, v5, v0
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v4, v1, v3 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v1, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v4, v1, v3 op_sel_hi:[1,0,1]
; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
+; GFX1200-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
+; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v4.l, v1.l
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v3.l
-; GFX1200-TRUE16-NEXT: global_store_b16 v1, v2, s[0:1]
+; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v3.l, v0.l, v4.l
+; GFX1200-TRUE16-NEXT: global_store_b16 v2, v3, s[0:1]
; GFX1200-TRUE16-NEXT: s_endpgm
;
; GFX1200-FAKE16-LABEL: unsafe_frem_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll
index 9a569fd127d73..14886857fa2f0 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll
@@ -102,7 +102,7 @@ define amdgpu_kernel void @cos_f16(ptr addrspace(1) %r, ptr addrspace(1) %a) {
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mul_f16_e32 v0.l, 0.15915494, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll
index ad690fe234bc7..1c035c4709bab 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll
@@ -96,7 +96,7 @@ define amdgpu_kernel void @rint_f16(
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_rndne_f16_e32 v0.l, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll
index d8f53af61b343..0a5df29922f45 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll
@@ -102,7 +102,7 @@ define amdgpu_kernel void @sin_f16(ptr addrspace(1) %r, ptr addrspace(1) %a) {
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mul_f16_e32 v0.l, 0.15915494, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
index 30d966eb1b7c2..14945be58876a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
@@ -23,7 +23,7 @@ define amdgpu_kernel void @sqrt_bf16(ptr addrspace(1) %r, ptr addrspace(1) %a) {
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s8, s2
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s9, s3
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-SDAG-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
+; GFX12-TRUE16-SDAG-NEXT: buffer_load_u16 v0, off, s[8:11], null
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s5, s1
; GFX12-TRUE16-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-SDAG-NEXT: v_sqrt_bf16_e32 v0.l, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
index bcaba060b812f..c9f4e1993cd73 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
@@ -129,7 +129,7 @@ define amdgpu_kernel void @sqrt_f16(
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s8, s2
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s9, s3
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-SDAG-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
+; GFX12-TRUE16-SDAG-NEXT: buffer_load_u16 v0, off, s[8:11], null
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s5, s1
; GFX12-TRUE16-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-SDAG-NEXT: v_sqrt_f16_e32 v0.l, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll
index 4664a4475d22a..9715cb0e3ee5b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll
@@ -95,7 +95,7 @@ define amdgpu_kernel void @trunc_f16(
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
+; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index 633c54f06a362..e95e1763ce30a 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -534,55 +534,29 @@ define amdgpu_kernel void @constant_load_v16i1(ptr addrspace(1) %out, ptr addrsp
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_load_v16i1:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_load_v16i1:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-TRUE16-LABEL: constant_load_v16i1:
-; GFX1250-TRUE16: ; %bb.0:
-; GFX1250-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-TRUE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-TRUE16-NEXT: v_nop
-; GFX1250-TRUE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
-; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX1250-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX1250-TRUE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_load_v16i1:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
;
-; GFX1250-FAKE16-LABEL: constant_load_v16i1:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-FAKE16-NEXT: v_nop
-; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX1250-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; GFX1250-FAKE16-NEXT: s_endpgm
+; GFX1250-LABEL: constant_load_v16i1:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX1250-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX1250-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
store <16 x i1> %load, ptr addrspace(1) %out
ret void
@@ -2240,32 +2214,31 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s6, 0xffff, s2
; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10001
; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10007
-; GFX12-NEXT: s_bfe_u32 s7, s2, 0x10009
-; GFX12-NEXT: s_bfe_u32 s8, s2, 0x1000d
-; GFX12-NEXT: s_and_b32 s9, s2, 1
-; GFX12-NEXT: s_bfe_u32 s10, s2, 0x1000a
-; GFX12-NEXT: s_bfe_u32 s2, s2, 0x1000c
-; GFX12-NEXT: s_bfe_u32 s11, s6, 0x10005
-; GFX12-NEXT: s_bfe_u32 s12, s6, 0x1000b
-; GFX12-NEXT: s_lshr_b32 s13, s6, 15
-; GFX12-NEXT: s_bfe_u32 s14, s6, 0x10002
-; GFX12-NEXT: s_bfe_u32 s15, s6, 0x10006
-; GFX12-NEXT: s_bfe_u32 s16, s6, 0x10004
-; GFX12-NEXT: s_bfe_u32 s17, s6, 0x10008
-; GFX12-NEXT: s_bfe_u32 s6, s6, 0x1000e
-; GFX12-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v1, s8
-; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s7
-; GFX12-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v2, s6
-; GFX12-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v4, s17
-; GFX12-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s5
-; GFX12-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v8, s16
-; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s15
-; GFX12-NEXT: v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v13, s4
-; GFX12-NEXT: v_mov_b32_e32 v14, s14
+; GFX12-NEXT: s_bfe_u32 s6, s2, 0x10005
+; GFX12-NEXT: s_bfe_u32 s7, s2, 0x1000b
+; GFX12-NEXT: s_bfe_u32 s8, s2, 0x10009
+; GFX12-NEXT: s_lshr_b32 s9, s2, 15
+; GFX12-NEXT: s_bfe_u32 s10, s2, 0x1000d
+; GFX12-NEXT: s_and_b32 s11, s2, 1
+; GFX12-NEXT: s_bfe_u32 s12, s2, 0x10002
+; GFX12-NEXT: s_bfe_u32 s13, s2, 0x10006
+; GFX12-NEXT: s_bfe_u32 s14, s2, 0x10004
+; GFX12-NEXT: s_bfe_u32 s15, s2, 0x1000a
+; GFX12-NEXT: s_bfe_u32 s16, s2, 0x10008
+; GFX12-NEXT: s_bfe_u32 s17, s2, 0x1000c
+; GFX12-NEXT: s_bfe_u32 s2, s2, 0x1000e
+; GFX12-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v1, s10
+; GFX12-NEXT: v_dual_mov_b32 v0, s17 :: v_dual_mov_b32 v3, s9
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v5, s8
+; GFX12-NEXT: v_dual_mov_b32 v4, s16 :: v_dual_mov_b32 v7, s7
+; GFX12-NEXT: v_dual_mov_b32 v6, s15 :: v_dual_mov_b32 v9, s6
+; GFX12-NEXT: v_dual_mov_b32 v8, s14 :: v_dual_mov_b32 v11, s5
+; GFX12-NEXT: v_dual_mov_b32 v10, s13 :: v_dual_mov_b32 v13, s4
+; GFX12-NEXT: v_dual_mov_b32 v12, s11 :: v_dual_mov_b32 v15, s3
+; GFX12-NEXT: v_mov_b32_e32 v14, s12
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX12-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
@@ -2283,32 +2256,31 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_and_b32 s6, 0xffff, s2
; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10001
; GFX1250-NEXT: s_bfe_u32 s5, s2, 0x10007
-; GFX1250-NEXT: s_bfe_u32 s7, s2, 0x10009
-; GFX1250-NEXT: s_bfe_u32 s8, s2, 0x1000d
-; GFX1250-NEXT: s_and_b32 s9, s2, 1
-; GFX1250-NEXT: s_bfe_u32 s10, s2, 0x1000a
-; GFX1250-NEXT: s_bfe_u32 s2, s2, 0x1000c
-; GFX1250-NEXT: s_bfe_u32 s11, s6, 0x10005
-; GFX1250-NEXT: s_bfe_u32 s12, s6, 0x1000b
-; GFX1250-NEXT: s_lshr_b32 s13, s6, 15
-; GFX1250-NEXT: s_bfe_u32 s14, s6, 0x10002
-; GFX1250-NEXT: s_bfe_u32 s15, s6, 0x10006
-; GFX1250-NEXT: s_bfe_u32 s16, s6, 0x10004
-; GFX1250-NEXT: s_bfe_u32 s17, s6, 0x10008
-; GFX1250-NEXT: s_bfe_u32 s6, s6, 0x1000e
-; GFX1250-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v0, s2
-; GFX1250-NEXT: v_dual_mov_b32 v1, s8 :: v_dual_mov_b32 v5, s7
-; GFX1250-NEXT: v_dual_mov_b32 v15, s3 :: v_dual_mov_b32 v2, s6
-; GFX1250-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v4, s17
-; GFX1250-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s5
-; GFX1250-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v8, s16
-; GFX1250-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s15
-; GFX1250-NEXT: v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v13, s4
-; GFX1250-NEXT: v_mov_b32_e32 v14, s14
+; GFX1250-NEXT: s_bfe_u32 s6, s2, 0x10005
+; GFX1250-NEXT: s_bfe_u32 s7, s2, 0x1000b
+; GFX1250-NEXT: s_bfe_u32 s8, s2, 0x10009
+; GFX1250-NEXT: s_lshr_b32 s9, s2, 15
+; GFX1250-NEXT: s_bfe_u32 s10, s2, 0x1000d
+; GFX1250-NEXT: s_and_b32 s11, s2, 1
+; GFX1250-NEXT: s_bfe_u32 s12, s2, 0x10002
+; GFX1250-NEXT: s_bfe_u32 s13, s2, 0x10006
+; GFX1250-NEXT: s_bfe_u32 s14, s2, 0x10004
+; GFX1250-NEXT: s_bfe_u32 s15, s2, 0x1000a
+; GFX1250-NEXT: s_bfe_u32 s16, s2, 0x10008
+; GFX1250-NEXT: s_bfe_u32 s17, s2, 0x1000c
+; GFX1250-NEXT: s_bfe_u32 s2, s2, 0x1000e
+; GFX1250-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v0, s17
+; GFX1250-NEXT: v_dual_mov_b32 v1, s10 :: v_dual_mov_b32 v2, s2
+; GFX1250-NEXT: v_dual_mov_b32 v3, s9 :: v_dual_mov_b32 v4, s16
+; GFX1250-NEXT: v_dual_mov_b32 v5, s8 :: v_dual_mov_b32 v6, s15
+; GFX1250-NEXT: v_dual_mov_b32 v7, s7 :: v_dual_mov_b32 v8, s14
+; GFX1250-NEXT: v_dual_mov_b32 v9, s6 :: v_dual_mov_b32 v10, s13
+; GFX1250-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v12, s11
+; GFX1250-NEXT: v_dual_mov_b32 v13, s4 :: v_dual_mov_b32 v14, s12
+; GFX1250-NEXT: v_mov_b32_e32 v15, s3
; GFX1250-NEXT: s_clause 0x3
; GFX1250-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX1250-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
@@ -7294,38 +7266,36 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000a
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s4
-; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10009
-; GFX12-NEXT: s_bfe_u32 s5, s3, 0x1000b
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, s5
-; GFX12-NEXT: s_bfe_u32 s5, s3, 0x10008
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000a
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s3
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000b
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10009
+; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, v1
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10008
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s5
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_lshr_b32 s4, s3, 15
-; GFX12-NEXT: s_bfe_u32 s5, s3, 0x1000e
-; GFX12-NEXT: s_bfe_u32 s3, s3, 0x10005
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: s_lshr_b32 s3, s2, 15
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000e
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s5
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000d
-; GFX12-NEXT: s_bfe_u32 s5, s2, 0x1000c
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000d
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000c
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:112
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s5
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10007
-; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10006
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10007
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10006
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s5
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
+; GFX12-NEXT: v_mov_b32_e32 v0, s4
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10005
; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10004
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
@@ -7356,51 +7326,47 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000a
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_mov_b32 v2, s4
-; GFX1250-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10009
-; GFX1250-NEXT: s_bfe_u32 s5, s3, 0x1000b
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v4, s5
-; GFX1250-NEXT: s_bfe_u32 s5, s3, 0x10008
-; GFX1250-NEXT: v_mov_b32_e32 v1, v3
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x1000a
+; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s3
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000b
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10009
+; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, v1
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10008
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-NEXT: s_lshr_b32 s4, s3, 15
-; GFX1250-NEXT: s_bfe_u32 s5, s3, 0x1000e
-; GFX1250-NEXT: s_bfe_u32 s3, s3, 0x10005
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:64
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
+; GFX1250-NEXT: s_lshr_b32 s3, s2, 15
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000e
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000d
-; GFX1250-NEXT: s_bfe_u32 s5, s2, 0x1000c
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:112
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x1000d
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000c
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:112
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10007
-; GFX1250-NEXT: s_bfe_u32 s5, s2, 0x10006
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10007
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10006
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10005
; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10004
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:48
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX1250-NEXT: v_mov_b32_e32 v0, s2
-; GFX1250-NEXT: s_bfe_u32 s2, s2, 0x10001
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:32
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
-; GFX1250-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX1250-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
+; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10001
+; GFX1250-NEXT: s_and_b32 s2, s2, 1
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:16
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: v_mov_b32_e32 v2, s2
-; GFX1250-NEXT: global_store_b128 v3, v[0:3], s[0:1]
+; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <16 x i1>, ptr addrspace(4) %in
%ext = zext <16 x i1> %load to <16 x i64>
@@ -7754,7 +7720,7 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12-TRUE16-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
; GFX12-TRUE16-NEXT: global_store_b128 v32, v[28:31], s[0:1]
; GFX12-TRUE16-NEXT: s_endpgm
-;
+
; GFX12-FAKE16-LABEL: constant_sextload_v16i1_to_v16i64:
; GFX12-FAKE16: ; %bb.0:
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
@@ -7826,6 +7792,7 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12-FAKE16-NEXT: global_store_b128 v32, v[28:31], s[0:1]
; GFX12-FAKE16-NEXT: s_endpgm
+
; GFX1250-LABEL: constant_sextload_v16i1_to_v16i64:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
index 895dedfff038e..9189fdbadd2c9 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -79,26 +79,15 @@ define amdgpu_kernel void @constant_load_i16(ptr addrspace(1) %out, ptr addrspac
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_load_i16:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_load_i16:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_load_i16:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
entry:
%ld = load i16, ptr addrspace(4) %in
store i16 %ld, ptr addrspace(1) %out
@@ -760,77 +749,47 @@ define amdgpu_kernel void @constant_load_v16i16_align2(ptr addrspace(4) %ptr0) #
; EG-NEXT: MOV * T2.X, literal.x,
; EG-NEXT: 0(0.000000e+00), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_load_v16i16_align2:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v8, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_clause 0x7
-; GFX12-TRUE16-NEXT: s_load_u16 s2, s[0:1], 0xc
-; GFX12-TRUE16-NEXT: s_load_u16 s3, s[0:1], 0x8
-; GFX12-TRUE16-NEXT: s_load_u16 s4, s[0:1], 0x4
-; GFX12-TRUE16-NEXT: s_load_u16 s5, s[0:1], 0x0
-; GFX12-TRUE16-NEXT: s_load_u16 s6, s[0:1], 0x1c
-; GFX12-TRUE16-NEXT: s_load_u16 s7, s[0:1], 0x18
-; GFX12-TRUE16-NEXT: s_load_u16 s8, s[0:1], 0x14
-; GFX12-TRUE16-NEXT: s_load_u16 s9, s[0:1], 0x10
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v3.l, s2
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v2.l, s3
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v1.l, s4
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s5
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v7.l, s6
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v6.l, s7
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v5.l, s8
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v4.l, s9
-; GFX12-TRUE16-NEXT: s_clause 0x7
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v7, v8, s[0:1] offset:30
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v6, v8, s[0:1] offset:26
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v5, v8, s[0:1] offset:22
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v4, v8, s[0:1] offset:18
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v1, v8, s[0:1] offset:6
-; GFX12-TRUE16-NEXT: global_load_d16_hi_b16 v0, v8, s[0:1] offset:2
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: global_store_b128 v[0:1], v[4:7], off
-; GFX12-TRUE16-NEXT: global_store_b128 v[0:1], v[0:3], off
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_load_v16i16_align2:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_clause 0x7
-; GFX12-FAKE16-NEXT: s_load_u16 s2, s[0:1], 0xc
-; GFX12-FAKE16-NEXT: s_load_u16 s3, s[0:1], 0x8
-; GFX12-FAKE16-NEXT: s_load_u16 s4, s[0:1], 0x4
-; GFX12-FAKE16-NEXT: s_load_u16 s5, s[0:1], 0x0
-; GFX12-FAKE16-NEXT: s_load_u16 s6, s[0:1], 0x1c
-; GFX12-FAKE16-NEXT: s_load_u16 s7, s[0:1], 0x18
-; GFX12-FAKE16-NEXT: s_load_u16 s8, s[0:1], 0x14
-; GFX12-FAKE16-NEXT: s_load_u16 s9, s[0:1], 0x10
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v8, 0 :: v_dual_mov_b32 v3, s2
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v1, s4
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s5 :: v_dual_mov_b32 v7, s6
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, s7 :: v_dual_mov_b32 v5, s8
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v4, s9
-; GFX12-FAKE16-NEXT: s_clause 0x7
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v3, v8, s[0:1] offset:14
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v7, v8, s[0:1] offset:30
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v6, v8, s[0:1] offset:26
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v5, v8, s[0:1] offset:22
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v4, v8, s[0:1] offset:18
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v2, v8, s[0:1] offset:10
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v1, v8, s[0:1] offset:6
-; GFX12-FAKE16-NEXT: global_load_d16_hi_b16 v0, v8, s[0:1] offset:2
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: global_store_b128 v[0:1], v[4:7], off
-; GFX12-FAKE16-NEXT: global_store_b128 v[0:1], v[0:3], off
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_load_v16i16_align2:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b64 s[0:1], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_clause 0xf
+; GFX12-NEXT: s_load_u16 s2, s[0:1], 0x0
+; GFX12-NEXT: s_load_u16 s3, s[0:1], 0x2
+; GFX12-NEXT: s_load_u16 s4, s[0:1], 0x4
+; GFX12-NEXT: s_load_u16 s5, s[0:1], 0x6
+; GFX12-NEXT: s_load_u16 s6, s[0:1], 0x8
+; GFX12-NEXT: s_load_u16 s7, s[0:1], 0xa
+; GFX12-NEXT: s_load_u16 s8, s[0:1], 0xc
+; GFX12-NEXT: s_load_u16 s9, s[0:1], 0xe
+; GFX12-NEXT: s_load_u16 s10, s[0:1], 0x10
+; GFX12-NEXT: s_load_u16 s11, s[0:1], 0x18
+; GFX12-NEXT: s_load_u16 s12, s[0:1], 0x1a
+; GFX12-NEXT: s_load_u16 s13, s[0:1], 0x1c
+; GFX12-NEXT: s_load_u16 s14, s[0:1], 0x1e
+; GFX12-NEXT: s_load_u16 s15, s[0:1], 0x12
+; GFX12-NEXT: s_load_u16 s16, s[0:1], 0x14
+; GFX12-NEXT: s_load_u16 s0, s[0:1], 0x16
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_pack_ll_b32_b16 s2, s2, s3
+; GFX12-NEXT: s_pack_ll_b32_b16 s4, s4, s5
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
+; GFX12-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s4
+; GFX12-NEXT: s_pack_ll_b32_b16 s6, s6, s7
+; GFX12-NEXT: s_pack_ll_b32_b16 s1, s8, s9
+; GFX12-NEXT: v_dual_mov_b32 v6, s6 :: v_dual_mov_b32 v7, s1
+; GFX12-NEXT: s_pack_ll_b32_b16 s5, s11, s12
+; GFX12-NEXT: s_pack_ll_b32_b16 s3, s13, s14
+; GFX12-NEXT: s_pack_ll_b32_b16 s7, s10, s15
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, s3
+; GFX12-NEXT: s_pack_ll_b32_b16 s0, s16, s0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_dual_mov_b32 v0, s7 :: v_dual_mov_b32 v1, s0
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_store_b128 v[0:1], v[0:3], off
+; GFX12-NEXT: global_store_b128 v[0:1], v[4:7], off
+; GFX12-NEXT: s_endpgm
entry:
%ld = load <16 x i16>, ptr addrspace(4) %ptr0, align 2
store <16 x i16> %ld, ptr addrspace(1) poison, align 32
@@ -5429,8 +5388,6 @@ define amdgpu_kernel void @constant_zextload_i16_to_i64(ptr addrspace(1) %out, p
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX12-NEXT: global_store_b64 v1, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
@@ -5521,9 +5478,8 @@ define amdgpu_kernel void @constant_sextload_i16_to_i64(ptr addrspace(1) %out, p
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_sext_i32_i16 s2, s2
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_ashr_i32 s3, s2, 31
+; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x100000
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT: v_mov_b32_e32 v0, s2
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
@@ -5608,8 +5564,6 @@ define amdgpu_kernel void @constant_zextload_v1i16_to_v1i64(ptr addrspace(1) %ou
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s2, s2, 0xffff
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
; GFX12-NEXT: global_store_b64 v1, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
@@ -5695,9 +5649,8 @@ define amdgpu_kernel void @constant_sextload_v1i16_to_v1i64(ptr addrspace(1) %ou
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_sext_i32_i16 s2, s2
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: s_ashr_i32 s3, s2, 31
+; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x100000
+; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
; GFX12-NEXT: v_mov_b32_e32 v0, s2
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
@@ -9012,3 +8965,6 @@ define amdgpu_kernel void @constant_sextload_v32i16_to_v32i64(ptr addrspace(1) %
; }
attributes #0 = { nounwind }
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; GFX12-FAKE16: {{.*}}
+; GFX12-TRUE16: {{.*}}
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
index ec1444d983dbc..fba7b61f960cc 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
@@ -178,26 +178,15 @@ define amdgpu_kernel void @constant_load_v2i8(ptr addrspace(1) %out, ptr addrspa
; EG-NEXT: LSHR * T1.X, KC0[2].Y, literal.x,
; EG-NEXT: 2(2.802597e-45), 0(0.000000e+00)
;
-; GFX12-TRUE16-LABEL: constant_load_v2i8:
-; GFX12-TRUE16: ; %bb.0: ; %entry
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: v_mov_b16_e32 v0.l, s2
-; GFX12-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-;
-; GFX12-FAKE16-LABEL: constant_load_v2i8:
-; GFX12-FAKE16: ; %bb.0: ; %entry
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: s_load_u16 s2, s[2:3], 0x0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
-; GFX12-FAKE16-NEXT: global_store_b16 v0, v1, s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
+; GFX12-LABEL: constant_load_v2i8:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: global_store_b16 v0, v1, s[0:1]
+; GFX12-NEXT: s_endpgm
entry:
%ld = load <2 x i8>, ptr addrspace(4) %in
store <2 x i8> %ld, ptr addrspace(1) %out
@@ -951,12 +940,11 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i32(ptr addrspace(1) %out
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX12-NEXT: s_and_b32 s2, s2, 0xff
-; GFX12-NEXT: s_lshr_b32 s3, s3, 8
+; GFX12-NEXT: s_and_b32 s3, s2, 0xff
+; GFX12-NEXT: s_lshr_b32 s2, s2, 8
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s3
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
+; GFX12-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v1, s2
+; GFX12-NEXT: v_mov_b32_e32 v0, s3
; GFX12-NEXT: global_store_b64 v2, v[0:1], s[0:1]
; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
@@ -5627,13 +5615,12 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i64(ptr addrspace(1) %out
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
-; GFX12-NEXT: s_and_b32 s2, s2, 0xff
+; GFX12-NEXT: s_and_b32 s3, s2, 0xff
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s2
-; GFX12-NEXT: s_lshr_b32 s3, s3, 8
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s3
+; GFX12-NEXT: s_lshr_b32 s2, s2, 8
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v2, s3 :: v_dual_mov_b32 v3, v1
+; GFX12-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, v1
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
@@ -5738,20 +5725,15 @@ define amdgpu_kernel void @constant_sextload_v2i8_to_v2i64(ptr addrspace(1) %out
; GFX12-LABEL: constant_sextload_v2i8_to_v2i64:
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: v_mov_b32_e32 v4, 0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v0, s2
-; GFX12-NEXT: s_lshr_b32 s2, s2, 8
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_lshr_b32 s4, s2, 8
; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x80000
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v3, s3
-; GFX12-NEXT: v_bfe_i32 v0, v0, 0, 8
-; GFX12-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX12-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x80000
+; GFX12-NEXT: v_dual_mov_b32 v4, 0 :: v_dual_mov_b32 v1, s3
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s5
+; GFX12-NEXT: v_mov_b32_e32 v2, s4
; GFX12-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX12-NEXT: s_endpgm
%load = load <2 x i8>, ptr addrspace(4) %in
@@ -9477,9 +9459,8 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i16(ptr addrspace(1) %out
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s3, 0xffff, s2
+; GFX12-NEXT: s_lshr_b32 s3, s2, 8
; GFX12-NEXT: s_and_b32 s2, s2, 0xff
-; GFX12-NEXT: s_lshr_b32 s3, s3, 8
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: s_pack_ll_b32_b16 s2, s2, s3
; GFX12-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s2
diff --git a/llvm/test/CodeGen/AMDGPU/min.ll b/llvm/test/CodeGen/AMDGPU/min.ll
index b403c6f3419b4..b6ca816550cc7 100644
--- a/llvm/test/CodeGen/AMDGPU/min.ll
+++ b/llvm/test/CodeGen/AMDGPU/min.ll
@@ -3324,14 +3324,11 @@ define amdgpu_kernel void @v_test_umin_ult_i16_multi_use(ptr addrspace(1) %out0,
; GFX1250-NEXT: s_load_b256 s[8:15], s[4:5], 0x0 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_load_u16 s0, s[14:15], 0x0
-; GFX1250-NEXT: s_load_u16 s1, s[12:13], 0x0
+; GFX1250-NEXT: s_load_u16 s0, s[12:13], 0x0
+; GFX1250-NEXT: s_load_u16 s1, s[14:15], 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_and_b32 s2, 0xffff, s0
-; GFX1250-NEXT: s_and_b32 s3, 0xffff, s1
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: s_cmp_lt_u32 s3, s2
-; GFX1250-NEXT: s_cselect_b32 s0, s1, s0
+; GFX1250-NEXT: s_cmp_lt_u32 s0, s1
+; GFX1250-NEXT: s_cselect_b32 s0, s0, s1
; GFX1250-NEXT: s_cselect_b32 s1, 1, 0
; GFX1250-NEXT: v_mov_b32_e32 v1, s0
; GFX1250-NEXT: v_mov_b32_e32 v2, s1
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
index 99b77b1517bd2..b1184b56f4b79 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
@@ -9,13 +9,11 @@ define amdgpu_kernel void @exp_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[V_EXP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_EXP_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[V_EXP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_EXP_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_EXP_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_EXP_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
@@ -32,13 +30,11 @@ define amdgpu_kernel void @log_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[V_LOG_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_LOG_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[V_LOG_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_LOG_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_LOG_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_LOG_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
@@ -55,13 +51,11 @@ define amdgpu_kernel void @rcp_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[V_RCP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RCP_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[V_RCP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RCP_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RCP_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RCP_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
@@ -78,13 +72,11 @@ define amdgpu_kernel void @rsq_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[V_RSQ_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RSQ_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[V_RSQ_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RSQ_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RSQ_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RSQ_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
@@ -101,13 +93,11 @@ define amdgpu_kernel void @sqrt_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
+ ; CHECK-NEXT: [[V_SQRT_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_SQRT_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[V_SQRT_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_SQRT_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
- ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SQRT_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SQRT_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
diff --git a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
index d8ce9f1726965..bb9dd7fce9c5e 100644
--- a/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
+++ b/llvm/test/CodeGen/AMDGPU/preload-implicit-kernargs.ll
@@ -1,8 +1,8 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -mtriple=amdgpu9.42--amdhsa < %s | FileCheck -check-prefixes=GFX942 %s
; RUN: llc -mtriple=amdgpu9.0a--amdhsa < %s | FileCheck -check-prefixes=GFX90a %s
-; RUN: llc -mtriple=amdgpu12.50--amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-FAKE16 %s
-; RUN: llc -mtriple=amdgpu12.50--amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250,GFX1250-REAL16 %s
+; RUN: llc -mtriple=amdgpu12.50--amdhsa -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX1250 %s
+; RUN: llc -mtriple=amdgpu12.50--amdhsa -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX1250 %s
define amdgpu_kernel void @preload_block_count_x(ptr addrspace(1) inreg %out) #0 {
; GFX942-LABEL: preload_block_count_x:
@@ -298,30 +298,17 @@ define amdgpu_kernel void @incorrect_type_i16_block_count_x(ptr addrspace(1) inr
; GFX90a-NEXT: global_store_short v0, v1, s[8:9]
; GFX90a-NEXT: s_endpgm
;
-; GFX1250-FAKE16-LABEL: incorrect_type_i16_block_count_x:
-; GFX1250-FAKE16: ; %bb.0:
-; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-FAKE16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-FAKE16-NEXT: v_nop
-; GFX1250-FAKE16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-FAKE16-NEXT: s_load_u16 s0, s[0:1], 0x8 nv
-; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
-; GFX1250-FAKE16-NEXT: global_store_b16 v0, v1, s[2:3]
-; GFX1250-FAKE16-NEXT: s_endpgm
-;
-; GFX1250-REAL16-LABEL: incorrect_type_i16_block_count_x:
-; GFX1250-REAL16: ; %bb.0:
-; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-REAL16-NEXT: s_mov_b64 s[64:65], 0
-; GFX1250-REAL16-NEXT: v_nop
-; GFX1250-REAL16-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
-; GFX1250-REAL16-NEXT: s_load_u16 s0, s[0:1], 0x8 nv
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
-; GFX1250-REAL16-NEXT: v_mov_b16_e32 v0.l, s0
-; GFX1250-REAL16-NEXT: global_store_b16 v1, v0, s[2:3]
-; GFX1250-REAL16-NEXT: s_endpgm
+; GFX1250-LABEL: incorrect_type_i16_block_count_x:
+; GFX1250: ; %bb.0:
+; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-NEXT: s_mov_b64 s[64:65], 0
+; GFX1250-NEXT: v_nop
+; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
+; GFX1250-NEXT: s_load_u16 s0, s[0:1], 0x8 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v1, s0
+; GFX1250-NEXT: global_store_b16 v0, v1, s[2:3]
+; GFX1250-NEXT: s_endpgm
%imp_arg_ptr = call ptr addrspace(4) @llvm.amdgcn.implicitarg.ptr()
%load = load i16, ptr addrspace(4) %imp_arg_ptr
store i16 %load, ptr addrspace(1) %out
>From bc57b40a2e7f2e75cb03975ac004b15bb9404836 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Mon, 28 Sep 2026 09:36:36 -0500
Subject: [PATCH 3/6] Remove tablegen patterns
Signed-off-by: John Lu <John.Lu at amd.com>
---
llvm/lib/Target/AMDGPU/SMInstructions.td | 21 +++++++--------------
1 file changed, 7 insertions(+), 14 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index 0f155fd898e98..19aeafe9b30cc 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -980,7 +980,7 @@ multiclass SMLoad_Pattern <string Instr, ValueType vt, bit immci = true> {
}
}
-multiclass ScalarSubwordLoadPat <string Instr, SDPatternOperator node, ValueType vt> {
+multiclass ScalarLoadWithExtensionPat <string Instr, SDPatternOperator node, ValueType vt> {
// 1. IMM offset
def : GCNPat <
(node (SMRDImm i64:$sbase, i32:$offset)),
@@ -1041,19 +1041,12 @@ multiclass ScalarBufferLoadIntrinsicPat <SDPatternOperator name, string Instr> {
// selector to prefer those.
let AddedComplexity = 100 in {
-// Extending subword loads
-defm : ScalarSubwordLoadPat <"S_LOAD_U8", smrd_extloadi8, i32>;
-defm : ScalarSubwordLoadPat <"S_LOAD_U8", smrd_zextloadi8, i32>;
-defm : ScalarSubwordLoadPat <"S_LOAD_I8", smrd_sextloadi8, i32>;
-defm : ScalarSubwordLoadPat <"S_LOAD_U16", smrd_extloadi16, i32>;
-defm : ScalarSubwordLoadPat <"S_LOAD_U16", smrd_zextloadi16, i32>;
-defm : ScalarSubwordLoadPat <"S_LOAD_I16", smrd_sextloadi16, i32>;
-
-// Non-extending subword loads
-foreach vt = Reg16Types.types in {
- defm : ScalarSubwordLoadPat <"S_LOAD_U16", smrd_load, vt>;
-}
-
+defm : ScalarLoadWithExtensionPat <"S_LOAD_U8", smrd_extloadi8, i32>;
+defm : ScalarLoadWithExtensionPat <"S_LOAD_U8", smrd_zextloadi8, i32>;
+defm : ScalarLoadWithExtensionPat <"S_LOAD_I8", smrd_sextloadi8, i32>;
+defm : ScalarLoadWithExtensionPat <"S_LOAD_U16", smrd_extloadi16, i32>;
+defm : ScalarLoadWithExtensionPat <"S_LOAD_U16", smrd_zextloadi16, i32>;
+defm : ScalarLoadWithExtensionPat <"S_LOAD_I16", smrd_sextloadi16, i32>;
defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_byte, "S_BUFFER_LOAD_I8">;
defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_ubyte, "S_BUFFER_LOAD_U8">;
defm : ScalarBufferLoadIntrinsicPat <SIsbuffer_load_short, "S_BUFFER_LOAD_I16">;
>From 2583df6212ec148b31a6682eb03714fe0dc80257 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Mon, 28 Sep 2026 11:12:31 -0500
Subject: [PATCH 4/6] Unify opc and node parameters
Signed-off-by: John Lu <John.Lu at amd.com>
---
llvm/include/llvm/CodeGen/TargetLowering.h | 7 +++----
llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp | 3 +--
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 9 ++++-----
llvm/lib/Target/AMDGPU/SIISelLowering.h | 2 +-
4 files changed, 9 insertions(+), 12 deletions(-)
diff --git a/llvm/include/llvm/CodeGen/TargetLowering.h b/llvm/include/llvm/CodeGen/TargetLowering.h
index 9d909c100ab19..6f7553cd460f8 100644
--- a/llvm/include/llvm/CodeGen/TargetLowering.h
+++ b/llvm/include/llvm/CodeGen/TargetLowering.h
@@ -4839,10 +4839,9 @@ class LLVM_ABI TargetLowering : public TargetLoweringBase {
return isTypeLegal(VT);
}
- /// Overload that provides context about the specific node being optimized.
- /// By default, calls the version without the node parameter.
- virtual bool isTypeDesirableForOp(unsigned Opc, EVT VT, SDNode *N) const {
- return isTypeDesirableForOp(Opc, VT);
+ /// Overload that takes the specific node being optimized.
+ virtual bool isTypeDesirableForOp(SDNode *N, EVT VT) const {
+ return isTypeDesirableForOp(N->getOpcode(), VT);
}
/// Return true if it is profitable for dag combiner to transform a floating
diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
index c43f6990fb18c..ea2955050332e 100644
--- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp
@@ -17970,8 +17970,7 @@ SDValue DAGCombiner::visitTRUNCATE(SDNode *N) {
// fold (truncate (load x)) -> (smaller load x)
// fold (truncate (srl (load x), c)) -> (smaller load (x+c/evtbits))
- if (!LegalTypes ||
- TLI.isTypeDesirableForOp(N0.getOpcode(), VT, N0.getNode())) {
+ if (!LegalTypes || TLI.isTypeDesirableForOp(N0.getNode(), VT)) {
if (SDValue Reduced = reduceLoadWidth(N))
return Reduced;
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index 4fef4d04de7f6..e1fa33bbcc06a 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -2505,14 +2505,13 @@ bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT) const {
return TargetLowering::isTypeDesirableForOp(Op, VT);
}
-bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT,
- SDNode *N) const {
+bool SITargetLowering::isTypeDesirableForOp(SDNode *N, EVT VT) const {
// Do not convert uniform i32 loads to 16-bit.
// Uniform 16-bit loads are legalized to i16 = trunc (zextload i16->i32)
// to match subword load patterns. Allowing conversion back to a 16-bit
// load would create an infinite loop.
- if (Subtarget->hasScalarSubwordLoads() && Op == ISD::LOAD && !VT.isVector() &&
- VT.getSizeInBits() == 16) {
+ if (Subtarget->hasScalarSubwordLoads() && N->getOpcode() == ISD::LOAD &&
+ !VT.isVector() && VT.getSizeInBits() == 16) {
auto *Load = dyn_cast<LoadSDNode>(N);
if (Load && Load->getValueType(0) == MVT::i32 && !Load->isDivergent() &&
AMDGPU::isUniformMMO(Load->getMemOperand())) {
@@ -2520,7 +2519,7 @@ bool SITargetLowering::isTypeDesirableForOp(unsigned Op, EVT VT,
}
}
- return isTypeDesirableForOp(Op, VT);
+ return isTypeDesirableForOp(N->getOpcode(), VT);
}
MachinePointerInfo
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 8fdc82db000a0..000017ec2ff69 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -417,7 +417,7 @@ class SITargetLowering final : public AMDGPUTargetLowering {
bool isExtractVecEltCheap(EVT VT, unsigned Index) const override;
bool isTypeDesirableForOp(unsigned Op, EVT VT) const override;
- bool isTypeDesirableForOp(unsigned Op, EVT VT, SDNode *N) const override;
+ bool isTypeDesirableForOp(SDNode *N, EVT VT) const override;
bool isOffsetFoldingLegal(const GlobalAddressSDNode *GA) const override;
>From 8a893453c2374666c2ef3e26df2fa8d9b4f2e580 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Mon, 28 Sep 2026 14:45:51 -0500
Subject: [PATCH 5/6] Use isUniformLoad to guard legalization
---
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp | 27 --
llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h | 1 -
llvm/lib/Target/AMDGPU/SIISelLowering.cpp | 27 +-
llvm/lib/Target/AMDGPU/SIISelLowering.h | 2 +
llvm/lib/Target/AMDGPU/SMInstructions.td | 5 +-
llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll | 4 +-
llvm/test/CodeGen/AMDGPU/fcmp.f16.ll | 88 ++---
llvm/test/CodeGen/AMDGPU/fmax3.ll | 48 +--
llvm/test/CodeGen/AMDGPU/fmaximum.ll | 6 +-
llvm/test/CodeGen/AMDGPU/fmin3.ll | 48 +--
llvm/test/CodeGen/AMDGPU/fminimum.ll | 6 +-
llvm/test/CodeGen/AMDGPU/frem.ll | 351 +++++++++---------
llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll | 2 +-
llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll | 2 +-
...-to-valu-pseudo-scalar-trans-f16-true16.ll | 50 ++-
19 files changed, 340 insertions(+), 335 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
index ff58f560314ab..86cb52485ee97 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.cpp
@@ -4755,33 +4755,6 @@ bool AMDGPUDAGToDAGISel::isVGPRImm(const SDNode * N) const {
return !AllUsesAcceptSReg && (Limit < 10);
}
-bool AMDGPUDAGToDAGISel::isUniformLoad(const SDNode *N) const {
- const auto *Ld = cast<LoadSDNode>(N);
- const MachineMemOperand *MMO = Ld->getMemOperand();
-
- // FIXME: We ought to able able to take the direct isDivergent result. We
- // cannot rely on the MMO for a uniformity check, and should stop using
- // it. This is a hack for 2 ways that the IR divergence analysis is superior
- // to the DAG divergence: Recognizing shift-of-workitem-id as always
- // uniform, and isSingleLaneExecution. These should be handled in the DAG
- // version, and then this can be dropped.
- if (Ld->isDivergent() && !AMDGPU::isUniformMMO(MMO))
- return false;
-
- return MMO->getSize().hasValue() &&
- Ld->getAlign() >=
- Align(std::min(MMO->getSize().getValue().getKnownMinValue(),
- uint64_t(4))) &&
- (MMO->isInvariant() ||
- (Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
- Ld->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
- (Subtarget->getScalarizeGlobalBehavior() &&
- Ld->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
- Ld->isSimple() &&
- static_cast<const SITargetLowering *>(getTargetLowering())
- ->isMemOpHasNoClobberedMemOperand(N)));
-}
-
void AMDGPUDAGToDAGISel::PostprocessISelDAG() {
const AMDGPUTargetLowering& Lowering =
*static_cast<const AMDGPUTargetLowering*>(getTargetLowering());
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
index 2bcfe530b3947..a0e6b547b90af 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelDAGToDAG.h
@@ -90,7 +90,6 @@ class AMDGPUDAGToDAGISel : public SelectionDAGISel {
}
bool isVGPRImm(const SDNode *N) const;
- bool isUniformLoad(const SDNode *N) const;
bool isUniformBr(const SDNode *N) const;
MachineSDNode *buildRegSequence16(SmallVectorImpl<SDValue> &Elts,
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
index e1fa33bbcc06a..84ba4dafc4766 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.cpp
@@ -2522,6 +2522,30 @@ bool SITargetLowering::isTypeDesirableForOp(SDNode *N, EVT VT) const {
return isTypeDesirableForOp(N->getOpcode(), VT);
}
+bool SITargetLowering::isUniformLoad(const LoadSDNode *Load) const {
+ const MachineMemOperand *MMO = Load->getMemOperand();
+
+ // FIXME: We ought to able able to take the direct isDivergent result. We
+ // cannot rely on the MMO for a uniformity check, and should stop using
+ // it. This is a hack for 2 ways that the IR divergence analysis is superior
+ // to the DAG divergence: Recognizing shift-of-workitem-id as always
+ // uniform, and isSingleLaneExecution. These should be handled in the DAG
+ // version, and then this can be dropped.
+ if (Load->isDivergent() && !AMDGPU::isUniformMMO(MMO))
+ return false;
+
+ return MMO->getSize().hasValue() &&
+ Load->getAlign() >=
+ Align(std::min(MMO->getSize().getValue().getKnownMinValue(),
+ uint64_t(4))) &&
+ (MMO->isInvariant() ||
+ (Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS ||
+ Load->getAddressSpace() == AMDGPUAS::CONSTANT_ADDRESS_32BIT) ||
+ (Load->getAddressSpace() == AMDGPUAS::GLOBAL_ADDRESS &&
+ Load->isSimple() && Subtarget->getScalarizeGlobalBehavior() &&
+ isMemOpHasNoClobberedMemOperand(Load)));
+}
+
MachinePointerInfo
SITargetLowering::getKernargSegmentPtrInfo(MachineFunction &MF) const {
// This isn't really a constant pool but close enough.
@@ -13624,9 +13648,10 @@ SDValue SITargetLowering::LowerLOAD(SDValue Op, SelectionDAG &DAG) const {
if (ExtType == ISD::NON_EXTLOAD && MemVT.getSizeInBits() < 32) {
// Legalize uniform 16-bit loads to i16 = trunc (zextload i16->i32)
// to match subword load patterns.
+ // Only do this for loads that can use scalar subword load instructions.
if (!MemVT.isVector() && MemVT.getSizeInBits() == 16 &&
isTypeLegal(MemVT) && Subtarget->hasScalarSubwordLoads() &&
- !Load->isDivergent() && AMDGPU::isUniformMMO(MMO)) {
+ isUniformLoad(Load)) {
SDValue Chain = Load->getChain();
SDValue BasePtr = Load->getBasePtr();
diff --git a/llvm/lib/Target/AMDGPU/SIISelLowering.h b/llvm/lib/Target/AMDGPU/SIISelLowering.h
index 000017ec2ff69..94d1714460fb3 100644
--- a/llvm/lib/Target/AMDGPU/SIISelLowering.h
+++ b/llvm/lib/Target/AMDGPU/SIISelLowering.h
@@ -54,6 +54,8 @@ class SITargetLowering final : public AMDGPUTargetLowering {
MachinePointerInfo getKernargSegmentPtrInfo(MachineFunction &MF) const;
+ bool isUniformLoad(const LoadSDNode *Load) const;
+
private:
SDValue lowerKernArgParameterPtr(SelectionDAG &DAG, const SDLoc &SL,
SDValue Chain, uint64_t Offset) const;
diff --git a/llvm/lib/Target/AMDGPU/SMInstructions.td b/llvm/lib/Target/AMDGPU/SMInstructions.td
index 19aeafe9b30cc..a4ef127a9029d 100644
--- a/llvm/lib/Target/AMDGPU/SMInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SMInstructions.td
@@ -828,7 +828,10 @@ def S_DCACHE_INV_VOL_ci : SMRD_Real_ci <0x1d, S_DCACHE_INV_VOL>;
// Scalar Memory Patterns
//===----------------------------------------------------------------------===//
-class SMRDLoadPat<PatFrag Op> : PatFrag <(ops node:$ptr), (Op node:$ptr), [{ return isUniformLoad(N);}]> {
+class SMRDLoadPat<PatFrag Op> : PatFrag <(ops node:$ptr), (Op node:$ptr), [{
+ return static_cast<const SITargetLowering *>(getTargetLowering())
+ ->isUniformLoad(cast<LoadSDNode>(N));
+}]> {
let GISelPredicateCode = [{
if (!MI.hasOneMemOperand())
return false;
diff --git a/llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll b/llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll
index 25a488f2b795f..7fe521102be99 100644
--- a/llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll
+++ b/llvm/test/CodeGen/AMDGPU/d16-write-vgpr32.ll
@@ -126,9 +126,9 @@ define amdgpu_kernel void @d16_two_order_group(ptr addrspace(3) %in1, ptr %in2)
; GFX12-NEXT: flat_load_d16_b16 v0, v[1:2]
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v3, s0
-; GFX12-NEXT: ds_load_u16 v3, v3
+; GFX12-NEXT: ds_load_u16_d16_hi v0, v3
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX12-NEXT: v_or_b16 v0.l, v3.l, v0.l
+; GFX12-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX12-NEXT: flat_store_b16 v[1:2], v0
; GFX12-NEXT: s_endpgm
%i16a = load i16, ptr addrspace(3) %in1, align 2
diff --git a/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll b/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
index 3685fb0d5d18f..1720f242fcc96 100644
--- a/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fcmp.f16.ll
@@ -131,13 +131,13 @@ define amdgpu_kernel void @fcmp_f16_lt(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_lt_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -303,9 +303,9 @@ define amdgpu_kernel void @fcmp_f16_lt_abs(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
@@ -483,13 +483,13 @@ define amdgpu_kernel void @fcmp_f16_eq(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_eq_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_eq_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -655,13 +655,13 @@ define amdgpu_kernel void @fcmp_f16_le(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_le_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -827,13 +827,13 @@ define amdgpu_kernel void @fcmp_f16_gt(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_gt_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -999,13 +999,13 @@ define amdgpu_kernel void @fcmp_f16_lg(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1171,13 +1171,13 @@ define amdgpu_kernel void @fcmp_f16_ge(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_ge_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_ge_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1343,13 +1343,13 @@ define amdgpu_kernel void @fcmp_f16_o(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_o_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1515,13 +1515,13 @@ define amdgpu_kernel void @fcmp_f16_u(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_u_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1687,13 +1687,13 @@ define amdgpu_kernel void @fcmp_f16_nge(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_nge_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -1859,13 +1859,13 @@ define amdgpu_kernel void @fcmp_f16_nlg(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_nlg_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_nlg_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -2031,13 +2031,13 @@ define amdgpu_kernel void @fcmp_f16_ngt(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_ngt_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_ngt_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -2203,13 +2203,13 @@ define amdgpu_kernel void @fcmp_f16_nle(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_nle_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -2375,13 +2375,13 @@ define amdgpu_kernel void @fcmp_f16_neq(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_neq_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_neq_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
@@ -2547,13 +2547,13 @@ define amdgpu_kernel void @fcmp_f16_nlt(
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_cmp_nlt_f16_e32 vcc_lo, v0.l, v1.l
+; GFX12-TRUE16-NEXT: v_cmp_nlt_f16_e32 vcc_lo, v0.l, v0.h
; GFX12-TRUE16-NEXT: v_cndmask_b32_e64 v0, 0, -1, vcc_lo
; GFX12-TRUE16-NEXT: buffer_store_b32 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/fmax3.ll b/llvm/test/CodeGen/AMDGPU/fmax3.ll
index 6618966a8e1ce..f34efb4f4af2b 100644
--- a/llvm/test/CodeGen/AMDGPU/fmax3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmax3.ll
@@ -696,15 +696,15 @@ define amdgpu_kernel void @test_fmax3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
; GFX12-TRUE16-NEXT: s_mov_b32 s17, s5
; GFX12-TRUE16-NEXT: s_mov_b32 s20, s6
; GFX12-TRUE16-NEXT: s_mov_b32 s21, s7
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v2, off, s[16:19], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[20:23], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_max3_num_f16 v0.l, v1.l, v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_max3_num_f16 v0.l, v0.l, v0.h, v1.l
; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -760,15 +760,15 @@ define amdgpu_kernel void @test_fmax3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
; GFX1250-TRUE16-NEXT: s_mov_b32 s17, s13
; GFX1250-TRUE16-NEXT: s_mov_b32 s20, s14
; GFX1250-TRUE16-NEXT: s_mov_b32 s21, s15
-; GFX1250-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_u16 v2, off, s[16:19], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_u16 v0, off, s[20:23], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s0, s8
; GFX1250-TRUE16-NEXT: s_mov_b32 s1, s9
-; GFX1250-TRUE16-NEXT: v_max3_num_f16 v0.l, v1.l, v2.l, v0.l
+; GFX1250-TRUE16-NEXT: v_max3_num_f16 v0.l, v0.l, v0.h, v1.l
; GFX1250-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-TRUE16-NEXT: s_endpgm
;
@@ -1044,24 +1044,24 @@ define amdgpu_kernel void @test_fmax3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
; GFX12-TRUE16-NEXT: s_mov_b32 s15, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s18, s10
; GFX12-TRUE16-NEXT: s_mov_b32 s19, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s22, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s23, s11
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s16, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s17, s5
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, s10
-; GFX12-TRUE16-NEXT: s_mov_b32 s7, s11
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_mov_b32 s20, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s21, s7
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v2, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_max3_num_f16 v0.l, v2.l, v1.l, v0.l
+; GFX12-TRUE16-NEXT: v_max3_num_f16 v0.l, v1.l, v0.l, v0.h
; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -1108,24 +1108,24 @@ define amdgpu_kernel void @test_fmax3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
; GFX1250-TRUE16-NEXT: s_mov_b32 s7, s3
; GFX1250-TRUE16-NEXT: s_mov_b32 s18, s2
; GFX1250-TRUE16-NEXT: s_mov_b32 s19, s3
+; GFX1250-TRUE16-NEXT: s_mov_b32 s22, s2
+; GFX1250-TRUE16-NEXT: s_mov_b32 s23, s3
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s4, s10
; GFX1250-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1250-TRUE16-NEXT: s_mov_b32 s16, s12
; GFX1250-TRUE16-NEXT: s_mov_b32 s17, s13
-; GFX1250-TRUE16-NEXT: s_mov_b32 s12, s14
-; GFX1250-TRUE16-NEXT: s_mov_b32 s13, s15
-; GFX1250-TRUE16-NEXT: s_mov_b32 s14, s2
-; GFX1250-TRUE16-NEXT: s_mov_b32 s15, s3
-; GFX1250-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: s_mov_b32 s20, s14
+; GFX1250-TRUE16-NEXT: s_mov_b32 s21, s15
+; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_u16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_u16 v2, off, s[12:15], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s0, s8
; GFX1250-TRUE16-NEXT: s_mov_b32 s1, s9
-; GFX1250-TRUE16-NEXT: v_max3_num_f16 v0.l, v2.l, v1.l, v0.l
+; GFX1250-TRUE16-NEXT: v_max3_num_f16 v0.l, v1.l, v0.l, v0.h
; GFX1250-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fmaximum.ll b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
index dffa28a5c52fd..213d6ffeb9f00 100644
--- a/llvm/test/CodeGen/AMDGPU/fmaximum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmaximum.ll
@@ -1492,11 +1492,11 @@ define amdgpu_kernel void @fmaximum_f16_move_to_valu(ptr addrspace(1) %out, ptr
; GFX12-SDAG-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-SDAG-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3] scope:SCOPE_SYS
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: global_load_u16 v2, v1, s[4:5] scope:SCOPE_SYS
+; GFX12-SDAG-TRUE16-NEXT: global_load_d16_hi_b16 v0, v1, s[4:5] scope:SCOPE_SYS
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_maximum_f16 v0.l, v0.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fmin3.ll b/llvm/test/CodeGen/AMDGPU/fmin3.ll
index bcd50ee8f47fd..c03e81e6b7129 100644
--- a/llvm/test/CodeGen/AMDGPU/fmin3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmin3.ll
@@ -696,15 +696,15 @@ define amdgpu_kernel void @test_fmin3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
; GFX12-TRUE16-NEXT: s_mov_b32 s17, s5
; GFX12-TRUE16-NEXT: s_mov_b32 s20, s6
; GFX12-TRUE16-NEXT: s_mov_b32 s21, s7
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v2, off, s[16:19], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[20:23], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_min3_num_f16 v0.l, v1.l, v2.l, v0.l
+; GFX12-TRUE16-NEXT: v_min3_num_f16 v0.l, v0.l, v0.h, v1.l
; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -760,15 +760,15 @@ define amdgpu_kernel void @test_fmin3_olt_0_f16(ptr addrspace(1) %out, ptr addrs
; GFX1250-TRUE16-NEXT: s_mov_b32 s17, s13
; GFX1250-TRUE16-NEXT: s_mov_b32 s20, s14
; GFX1250-TRUE16-NEXT: s_mov_b32 s21, s15
-; GFX1250-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_u16 v2, off, s[16:19], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_u16 v0, off, s[20:23], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s0, s8
; GFX1250-TRUE16-NEXT: s_mov_b32 s1, s9
-; GFX1250-TRUE16-NEXT: v_min3_num_f16 v0.l, v1.l, v2.l, v0.l
+; GFX1250-TRUE16-NEXT: v_min3_num_f16 v0.l, v0.l, v0.h, v1.l
; GFX1250-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-TRUE16-NEXT: s_endpgm
;
@@ -1044,24 +1044,24 @@ define amdgpu_kernel void @test_fmin3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
; GFX12-TRUE16-NEXT: s_mov_b32 s15, s11
; GFX12-TRUE16-NEXT: s_mov_b32 s18, s10
; GFX12-TRUE16-NEXT: s_mov_b32 s19, s11
+; GFX12-TRUE16-NEXT: s_mov_b32 s22, s10
+; GFX12-TRUE16-NEXT: s_mov_b32 s23, s11
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s12, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s13, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s16, s4
; GFX12-TRUE16-NEXT: s_mov_b32 s17, s5
-; GFX12-TRUE16-NEXT: s_mov_b32 s4, s6
-; GFX12-TRUE16-NEXT: s_mov_b32 s5, s7
-; GFX12-TRUE16-NEXT: s_mov_b32 s6, s10
-; GFX12-TRUE16-NEXT: s_mov_b32 s7, s11
-; GFX12-TRUE16-NEXT: buffer_load_u16 v1, off, s[12:15], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: s_mov_b32 s20, s6
+; GFX12-TRUE16-NEXT: s_mov_b32 s21, s7
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[12:15], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v2, off, s[4:7], null scope:SCOPE_SYS
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s0
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s1
-; GFX12-TRUE16-NEXT: v_min3_num_f16 v0.l, v2.l, v1.l, v0.l
+; GFX12-TRUE16-NEXT: v_min3_num_f16 v0.l, v1.l, v0.l, v0.h
; GFX12-TRUE16-NEXT: buffer_store_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_endpgm
;
@@ -1108,24 +1108,24 @@ define amdgpu_kernel void @test_fmin3_olt_1_f16(ptr addrspace(1) %out, ptr addrs
; GFX1250-TRUE16-NEXT: s_mov_b32 s7, s3
; GFX1250-TRUE16-NEXT: s_mov_b32 s18, s2
; GFX1250-TRUE16-NEXT: s_mov_b32 s19, s3
+; GFX1250-TRUE16-NEXT: s_mov_b32 s22, s2
+; GFX1250-TRUE16-NEXT: s_mov_b32 s23, s3
; GFX1250-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s4, s10
; GFX1250-TRUE16-NEXT: s_mov_b32 s5, s11
; GFX1250-TRUE16-NEXT: s_mov_b32 s16, s12
; GFX1250-TRUE16-NEXT: s_mov_b32 s17, s13
-; GFX1250-TRUE16-NEXT: s_mov_b32 s12, s14
-; GFX1250-TRUE16-NEXT: s_mov_b32 s13, s15
-; GFX1250-TRUE16-NEXT: s_mov_b32 s14, s2
-; GFX1250-TRUE16-NEXT: s_mov_b32 s15, s3
-; GFX1250-TRUE16-NEXT: buffer_load_u16 v1, off, s[4:7], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: s_mov_b32 s20, s14
+; GFX1250-TRUE16-NEXT: s_mov_b32 s21, s15
+; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[4:7], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_u16 v0, off, s[16:19], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_d16_hi_b16 v0, off, s[16:19], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-TRUE16-NEXT: buffer_load_u16 v2, off, s[12:15], null scope:SCOPE_SYS
+; GFX1250-TRUE16-NEXT: buffer_load_d16_b16 v1, off, s[20:23], null scope:SCOPE_SYS
; GFX1250-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-TRUE16-NEXT: s_mov_b32 s0, s8
; GFX1250-TRUE16-NEXT: s_mov_b32 s1, s9
-; GFX1250-TRUE16-NEXT: v_min3_num_f16 v0.l, v2.l, v1.l, v0.l
+; GFX1250-TRUE16-NEXT: v_min3_num_f16 v0.l, v1.l, v0.l, v0.h
; GFX1250-TRUE16-NEXT: buffer_store_b16 v0, off, s[0:3], null
; GFX1250-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/fminimum.ll b/llvm/test/CodeGen/AMDGPU/fminimum.ll
index 11acac09aa30f..094dd76e0f945 100644
--- a/llvm/test/CodeGen/AMDGPU/fminimum.ll
+++ b/llvm/test/CodeGen/AMDGPU/fminimum.ll
@@ -1492,11 +1492,11 @@ define amdgpu_kernel void @fminimum_f16_move_to_valu(ptr addrspace(1) %out, ptr
; GFX12-SDAG-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-SDAG-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3] scope:SCOPE_SYS
+; GFX12-SDAG-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3] scope:SCOPE_SYS
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: global_load_u16 v2, v1, s[4:5] scope:SCOPE_SYS
+; GFX12-SDAG-TRUE16-NEXT: global_load_d16_hi_b16 v0, v1, s[4:5] scope:SCOPE_SYS
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v2.l
+; GFX12-SDAG-TRUE16-NEXT: v_minimum_f16 v0.l, v0.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
; GFX12-SDAG-TRUE16-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index e2ea47c6b27fb..f202227662c40 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -978,249 +978,244 @@ define amdgpu_kernel void @frem_f16(ptr addrspace(1) %out, ptr addrspace(1) %in1
; GFX1200-TRUE16-LABEL: frem_f16:
; GFX1200-TRUE16: ; %bb.0:
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1200-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX1200-TRUE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1200-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
+; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1200-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX1200-TRUE16-NEXT: global_load_u16 v0, v0, s[4:5] offset:8
+; GFX1200-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[10:11]
+; GFX1200-TRUE16-NEXT: global_load_d16_b16 v1, v1, s[0:1] offset:8
; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x1
-; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s6, v1
+; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s3, v0
-; GFX1200-TRUE16-NEXT: s_and_b32 s2, s6, 0x7fff
-; GFX1200-TRUE16-NEXT: s_and_b32 s4, s3, 0x7fff
-; GFX1200-TRUE16-NEXT: s_cvt_f32_f16 s5, s2
-; GFX1200-TRUE16-NEXT: s_cvt_f32_f16 s4, s4
+; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1200-TRUE16-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX1200-TRUE16-NEXT: s_and_b32 s2, s1, 0x7fff
+; GFX1200-TRUE16-NEXT: s_cvt_f32_f16 s1, s0
+; GFX1200-TRUE16-NEXT: s_cvt_f32_f16 s0, s2
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1200-TRUE16-NEXT: s_cmp_ngt_f32 s5, s4
+; GFX1200-TRUE16-NEXT: s_cmp_ngt_f32 s1, s0
; GFX1200-TRUE16-NEXT: s_cbranch_scc0 .LBB0_2
; GFX1200-TRUE16-NEXT: ; %bb.1: ; %frem.else
-; GFX1200-TRUE16-NEXT: s_cmp_eq_f32 s5, s4
-; GFX1200-TRUE16-NEXT: v_and_b16 v0.l, 0x8000, s6
-; GFX1200-TRUE16-NEXT: s_cselect_b32 s7, -1, 0
-; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_cndmask_b16 v0.l, s6, v0.l, s7
-; GFX1200-TRUE16-NEXT: s_mov_b32 s6, 0
+; GFX1200-TRUE16-NEXT: s_cmp_eq_f32 s1, s0
+; GFX1200-TRUE16-NEXT: v_and_b16 v0.h, 0x8000, v0.l
+; GFX1200-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX1200-TRUE16-NEXT: v_cndmask_b16 v2.l, v0.l, v0.h, s2
+; GFX1200-TRUE16-NEXT: s_mov_b32 s2, 0
; GFX1200-TRUE16-NEXT: s_branch .LBB0_3
; GFX1200-TRUE16-NEXT: .LBB0_2:
-; GFX1200-TRUE16-NEXT: s_mov_b32 s6, -1
-; GFX1200-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX1200-TRUE16-NEXT: s_mov_b32 s2, -1
+; GFX1200-TRUE16-NEXT: ; implicit-def: $vgpr2
; GFX1200-TRUE16-NEXT: .LBB0_3: ; %Flow18
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_and_b32 s6, s6, exec_lo
-; GFX1200-TRUE16-NEXT: s_cselect_b32 s6, 1, 0
+; GFX1200-TRUE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1200-TRUE16-NEXT: s_cselect_b32 s2, 1, 0
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_cmp_lg_u32 s6, 1
+; GFX1200-TRUE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX1200-TRUE16-NEXT: s_cbranch_scc1 .LBB0_8
; GFX1200-TRUE16-NEXT: ; %bb.4: ; %frem.compute
-; GFX1200-TRUE16-NEXT: v_frexp_mant_f32_e32 v1, s4
-; GFX1200-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v3, s4
-; GFX1200-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v2, s5
-; GFX1200-TRUE16-NEXT: v_frexp_mant_f32_e32 v0, s5
+; GFX1200-TRUE16-NEXT: v_frexp_mant_f32_e32 v3, s0
+; GFX1200-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v5, s0
+; GFX1200-TRUE16-NEXT: v_frexp_exp_i32_f32_e32 v4, s1
+; GFX1200-TRUE16-NEXT: v_frexp_mant_f32_e32 v2, s1
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-TRUE16-NEXT: v_ldexp_f32 v1, v1, 1
-; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s4, v3
-; GFX1200-TRUE16-NEXT: v_add_nc_u32_e32 v3, -1, v3
-; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s5, v2
-; GFX1200-TRUE16-NEXT: v_ldexp_f32 v0, v0, 11
-; GFX1200-TRUE16-NEXT: v_div_scale_f32 v4, null, v1, v1, 1.0
+; GFX1200-TRUE16-NEXT: v_ldexp_f32 v3, v3, 1
+; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s0, v5
+; GFX1200-TRUE16-NEXT: v_add_nc_u32_e32 v5, -1, v5
+; GFX1200-TRUE16-NEXT: v_readfirstlane_b32 s1, v4
+; GFX1200-TRUE16-NEXT: v_ldexp_f32 v2, v2, 11
+; GFX1200-TRUE16-NEXT: v_div_scale_f32 v6, null, v3, v3, 1.0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1200-TRUE16-NEXT: v_not_b32_e32 v3, v3
-; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v5, v4
+; GFX1200-TRUE16-NEXT: v_not_b32_e32 v5, v5
+; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v7, v6
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(TRANS32_DEP_1)
-; GFX1200-TRUE16-NEXT: v_add_nc_u32_e32 v3, v3, v2
-; GFX1200-TRUE16-NEXT: v_div_scale_f32 v2, vcc_lo, 1.0, v1, 1.0
+; GFX1200-TRUE16-NEXT: v_add_nc_u32_e32 v5, v5, v4
+; GFX1200-TRUE16-NEXT: v_div_scale_f32 v4, vcc_lo, 1.0, v3, 1.0
; GFX1200-TRUE16-NEXT: s_denorm_mode 15
-; GFX1200-TRUE16-NEXT: v_fma_f32 v6, -v4, v5, 1.0
+; GFX1200-TRUE16-NEXT: v_fma_f32 v8, -v6, v7, 1.0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v5, v6, v5
-; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v6, v2, v5
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v7, v8, v7
+; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v8, v4, v7
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_f32 v7, -v4, v6, v2
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v6, v7, v5
+; GFX1200-TRUE16-NEXT: v_fma_f32 v9, -v6, v8, v4
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v8, v9, v7
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_f32 v2, -v4, v6, v2
+; GFX1200-TRUE16-NEXT: v_fma_f32 v4, -v6, v8, v4
; GFX1200-TRUE16-NEXT: s_denorm_mode 12
-; GFX1200-TRUE16-NEXT: v_div_fmas_f32 v2, v2, v5, v6
-; GFX1200-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v3
+; GFX1200-TRUE16-NEXT: v_div_fmas_f32 v4, v4, v7, v8
+; GFX1200-TRUE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v5
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1200-TRUE16-NEXT: v_div_fixup_f32 v2, v2, v1, 1.0
+; GFX1200-TRUE16-NEXT: v_div_fixup_f32 v4, v4, v3, 1.0
; GFX1200-TRUE16-NEXT: s_cbranch_vccnz .LBB0_7
; GFX1200-TRUE16-NEXT: ; %bb.5: ; %frem.loop_body.preheader
-; GFX1200-TRUE16-NEXT: s_sub_co_i32 s4, s5, s4
+; GFX1200-TRUE16-NEXT: s_sub_co_i32 s0, s1, s0
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_add_co_i32 s4, s4, 11
+; GFX1200-TRUE16-NEXT: s_add_co_i32 s0, s0, 11
; GFX1200-TRUE16-NEXT: .LBB0_6: ; %frem.loop_body
; GFX1200-TRUE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v3, v0, v2
+; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v5, v2, v4
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_add_co_i32 s4, s4, -11
+; GFX1200-TRUE16-NEXT: s_add_co_i32 s0, s0, -11
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-TRUE16-NEXT: s_cmp_gt_i32 s4, 11
-; GFX1200-TRUE16-NEXT: v_rndne_f32_e32 v3, v3
+; GFX1200-TRUE16-NEXT: s_cmp_gt_i32 s0, 11
+; GFX1200-TRUE16-NEXT: v_rndne_f32_e32 v5, v5
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v0, v3, v1
+; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v2, v5, v3
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v0
-; GFX1200-TRUE16-NEXT: v_add_f32_e32 v3, v0, v1
+; GFX1200-TRUE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v2
+; GFX1200-TRUE16-NEXT: v_add_f32_e32 v5, v2, v3
; GFX1200-TRUE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-TRUE16-NEXT: v_cndmask_b32_e32 v0, v0, v3, vcc_lo
+; GFX1200-TRUE16-NEXT: v_cndmask_b32_e32 v2, v2, v5, vcc_lo
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_ldexp_f32 v0, v0, 11
+; GFX1200-TRUE16-NEXT: v_ldexp_f32 v2, v2, 11
; GFX1200-TRUE16-NEXT: s_cbranch_scc1 .LBB0_6
; GFX1200-TRUE16-NEXT: .LBB0_7: ; %frem.loop_exit
-; GFX1200-TRUE16-NEXT: ; implicit-def: $vgpr0
+; GFX1200-TRUE16-NEXT: ; implicit-def: $vgpr2
; GFX1200-TRUE16-NEXT: .LBB0_8: ; %Flow19
-; GFX1200-TRUE16-NEXT: s_cmp_lg_f16 s3, 0
-; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
-; GFX1200-TRUE16-NEXT: s_cselect_b32 s3, -1, 0
-; GFX1200-TRUE16-NEXT: s_cmp_nge_f16 s2, 0x7c00
-; GFX1200-TRUE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(SALU_CYCLE_1)
-; GFX1200-TRUE16-NEXT: s_and_b32 s2, s2, s3
-; GFX1200-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v0.l, s2
-; GFX1200-TRUE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1200-TRUE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x7fff, v0
+; GFX1200-TRUE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0, v1.l
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-TRUE16-NEXT: v_cmp_nle_f16_e64 s0, 0x7c00, v0.l
+; GFX1200-TRUE16-NEXT: s_and_b32 s0, s0, vcc_lo
+; GFX1200-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX1200-TRUE16-NEXT: v_cndmask_b16 v0.l, 0x7e00, v2.l, s0
+; GFX1200-TRUE16-NEXT: global_store_b16 v3, v0, s[8:9]
; GFX1200-TRUE16-NEXT: s_endpgm
;
; GFX1200-FAKE16-LABEL: frem_f16:
; GFX1200-FAKE16: ; %bb.0:
; GFX1200-FAKE16-NEXT: s_clause 0x1
-; GFX1200-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX1200-FAKE16-NEXT: s_load_b64 s[6:7], s[4:5], 0x34
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX1200-FAKE16-NEXT: s_load_b128 s[8:11], s[4:5], 0x24
+; GFX1200-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x34
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1200-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-FAKE16-NEXT: s_clause 0x1
-; GFX1200-FAKE16-NEXT: global_load_u16 v1, v0, s[2:3]
-; GFX1200-FAKE16-NEXT: global_load_u16 v0, v0, s[6:7] offset:8
+; GFX1200-FAKE16-NEXT: global_load_u16 v0, v1, s[10:11]
+; GFX1200-FAKE16-NEXT: global_load_u16 v1, v1, s[0:1] offset:8
; GFX1200-FAKE16-NEXT: s_wait_loadcnt 0x1
-; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s4, v1
+; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s0, v0
; GFX1200-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s3, v0
-; GFX1200-FAKE16-NEXT: s_and_b32 s2, s4, 0x7fff
-; GFX1200-FAKE16-NEXT: s_and_b32 s5, s3, 0x7fff
-; GFX1200-FAKE16-NEXT: s_cvt_f32_f16 s6, s2
-; GFX1200-FAKE16-NEXT: s_cvt_f32_f16 s5, s5
+; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s1, v1
+; GFX1200-FAKE16-NEXT: s_and_b32 s0, s0, 0x7fff
+; GFX1200-FAKE16-NEXT: s_and_b32 s2, s1, 0x7fff
+; GFX1200-FAKE16-NEXT: s_cvt_f32_f16 s1, s0
+; GFX1200-FAKE16-NEXT: s_cvt_f32_f16 s0, s2
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_3)
-; GFX1200-FAKE16-NEXT: s_cmp_ngt_f32 s6, s5
+; GFX1200-FAKE16-NEXT: s_cmp_ngt_f32 s1, s0
; GFX1200-FAKE16-NEXT: s_cbranch_scc0 .LBB0_2
; GFX1200-FAKE16-NEXT: ; %bb.1: ; %frem.else
-; GFX1200-FAKE16-NEXT: s_and_b32 s7, 0x8000, s4
-; GFX1200-FAKE16-NEXT: s_cmp_eq_f32 s6, s5
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v0, s7
-; GFX1200-FAKE16-NEXT: s_mov_b32 s7, 0
+; GFX1200-FAKE16-NEXT: s_cmp_eq_f32 s1, s0
+; GFX1200-FAKE16-NEXT: v_and_b32_e32 v2, 0x8000, v0
+; GFX1200-FAKE16-NEXT: s_mov_b32 s2, 0
; GFX1200-FAKE16-NEXT: s_cselect_b32 vcc_lo, -1, 0
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v0, s4, v0, vcc_lo
+; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v2, v0, v2, vcc_lo
; GFX1200-FAKE16-NEXT: s_branch .LBB0_3
; GFX1200-FAKE16-NEXT: .LBB0_2:
-; GFX1200-FAKE16-NEXT: s_mov_b32 s7, -1
-; GFX1200-FAKE16-NEXT: ; implicit-def: $vgpr0
+; GFX1200-FAKE16-NEXT: s_mov_b32 s2, -1
+; GFX1200-FAKE16-NEXT: ; implicit-def: $vgpr2
; GFX1200-FAKE16-NEXT: .LBB0_3: ; %Flow18
-; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: s_and_b32 s7, s7, exec_lo
-; GFX1200-FAKE16-NEXT: s_cselect_b32 s7, 1, 0
-; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: s_cmp_lg_u32 s7, 1
+; GFX1200-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
+; GFX1200-FAKE16-NEXT: s_and_b32 s2, s2, exec_lo
+; GFX1200-FAKE16-NEXT: s_cselect_b32 s2, 1, 0
+; GFX1200-FAKE16-NEXT: s_cmp_lg_u32 s2, 1
; GFX1200-FAKE16-NEXT: s_cbranch_scc1 .LBB0_9
; GFX1200-FAKE16-NEXT: ; %bb.4: ; %frem.compute
-; GFX1200-FAKE16-NEXT: v_frexp_mant_f32_e32 v1, s5
-; GFX1200-FAKE16-NEXT: v_frexp_mant_f32_e32 v0, s6
-; GFX1200-FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v3, s6
+; GFX1200-FAKE16-NEXT: v_frexp_mant_f32_e32 v3, s0
+; GFX1200-FAKE16-NEXT: v_frexp_mant_f32_e32 v2, s1
+; GFX1200-FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v5, s1
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v1, v1, 1
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v2, v0, 11
-; GFX1200-FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v0, s5
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v3, v3, 1
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v4, v2, 11
+; GFX1200-FAKE16-NEXT: v_frexp_exp_i32_f32_e32 v2, s0
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s6, v3
-; GFX1200-FAKE16-NEXT: v_div_scale_f32 v5, null, v1, v1, 1.0
+; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s1, v5
+; GFX1200-FAKE16-NEXT: v_div_scale_f32 v7, null, v3, v3, 1.0
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s5, v0
-; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v0, -1, v0
-; GFX1200-FAKE16-NEXT: v_rcp_f32_e32 v6, v5
+; GFX1200-FAKE16-NEXT: v_readfirstlane_b32 s0, v2
+; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v2, -1, v2
+; GFX1200-FAKE16-NEXT: v_rcp_f32_e32 v8, v7
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_not_b32_e32 v4, v0
-; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v4, v4, v3
-; GFX1200-FAKE16-NEXT: v_div_scale_f32 v3, vcc_lo, 1.0, v1, 1.0
+; GFX1200-FAKE16-NEXT: v_not_b32_e32 v6, v2
+; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v6, v6, v5
+; GFX1200-FAKE16-NEXT: v_div_scale_f32 v5, vcc_lo, 1.0, v3, 1.0
; GFX1200-FAKE16-NEXT: s_denorm_mode 15
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(TRANS32_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_fma_f32 v7, -v5, v6, 1.0
-; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v6, v7, v6
+; GFX1200-FAKE16-NEXT: v_fma_f32 v9, -v7, v8, 1.0
+; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v8, v9, v8
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v7, v3, v6
-; GFX1200-FAKE16-NEXT: v_fma_f32 v8, -v5, v7, v3
+; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v9, v5, v8
+; GFX1200-FAKE16-NEXT: v_fma_f32 v10, -v7, v9, v5
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v7, v8, v6
-; GFX1200-FAKE16-NEXT: v_fma_f32 v3, -v5, v7, v3
+; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v9, v10, v8
+; GFX1200-FAKE16-NEXT: v_fma_f32 v5, -v7, v9, v5
; GFX1200-FAKE16-NEXT: s_denorm_mode 12
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX1200-FAKE16-NEXT: v_div_fmas_f32 v3, v3, v6, v7
-; GFX1200-FAKE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v4
-; GFX1200-FAKE16-NEXT: v_div_fixup_f32 v3, v3, v1, 1.0
+; GFX1200-FAKE16-NEXT: v_div_fmas_f32 v5, v5, v8, v9
+; GFX1200-FAKE16-NEXT: v_cmp_gt_i32_e32 vcc_lo, 12, v6
+; GFX1200-FAKE16-NEXT: v_div_fixup_f32 v5, v5, v3, 1.0
; GFX1200-FAKE16-NEXT: s_cbranch_vccnz .LBB0_8
; GFX1200-FAKE16-NEXT: ; %bb.5: ; %frem.loop_body.preheader
-; GFX1200-FAKE16-NEXT: s_sub_co_i32 s5, s6, s5
+; GFX1200-FAKE16-NEXT: s_sub_co_i32 s0, s1, s0
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: s_add_co_i32 s5, s5, 11
+; GFX1200-FAKE16-NEXT: s_add_co_i32 s0, s0, 11
; GFX1200-FAKE16-NEXT: .LBB0_6: ; %frem.loop_body
; GFX1200-FAKE16-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v5, v2
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v7, v4
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: s_add_co_i32 s5, s5, -11
+; GFX1200-FAKE16-NEXT: s_add_co_i32 s0, s0, -11
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: s_cmp_gt_i32 s5, 11
-; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v2, v5, v3
+; GFX1200-FAKE16-NEXT: s_cmp_gt_i32 s0, 11
+; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v4, v7, v5
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_rndne_f32_e32 v2, v2
-; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v2, 0x80000000, v2
+; GFX1200-FAKE16-NEXT: v_rndne_f32_e32 v4, v4
+; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v4, 0x80000000, v4
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_fma_f32 v2, v2, v1, v5
-; GFX1200-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v2
-; GFX1200-FAKE16-NEXT: v_add_f32_e32 v4, v2, v1
+; GFX1200-FAKE16-NEXT: v_fma_f32 v4, v4, v3, v7
+; GFX1200-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v4
+; GFX1200-FAKE16-NEXT: v_add_f32_e32 v6, v4, v3
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v2, v2, v4, vcc_lo
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v2, v2, 11
+; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v4, v4, v6, vcc_lo
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v4, v4, 11
; GFX1200-FAKE16-NEXT: s_cbranch_scc1 .LBB0_6
; GFX1200-FAKE16-NEXT: ; %bb.7: ; %Flow
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v4, s5
-; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v2, v5
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v6, s0
+; GFX1200-FAKE16-NEXT: v_mov_b32_e32 v4, v7
; GFX1200-FAKE16-NEXT: .LBB0_8: ; %frem.loop_exit
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v4, -10, v4
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v2, v2, v4
+; GFX1200-FAKE16-NEXT: v_add_nc_u32_e32 v6, -10, v6
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v4, v4, v6
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v3, v2, v3
-; GFX1200-FAKE16-NEXT: v_rndne_f32_e32 v3, v3
+; GFX1200-FAKE16-NEXT: v_mul_f32_e32 v5, v4, v5
+; GFX1200-FAKE16-NEXT: v_rndne_f32_e32 v5, v5
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v3, 0x80000000, v3
-; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v2, v3, v1
+; GFX1200-FAKE16-NEXT: v_xor_b32_e32 v5, 0x80000000, v5
+; GFX1200-FAKE16-NEXT: v_fmac_f32_e32 v4, v5, v3
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v2
-; GFX1200-FAKE16-NEXT: v_add_f32_e32 v1, v2, v1
+; GFX1200-FAKE16-NEXT: v_cmp_gt_f32_e32 vcc_lo, 0, v4
+; GFX1200-FAKE16-NEXT: v_add_f32_e32 v3, v4, v3
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v1, v2, v1, vcc_lo
+; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v3, v4, v3, vcc_lo
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_ldexp_f32 v0, v1, v0
-; GFX1200-FAKE16-NEXT: v_cvt_f16_f32_e32 v0, v0
+; GFX1200-FAKE16-NEXT: v_ldexp_f32 v2, v3, v2
+; GFX1200-FAKE16-NEXT: v_cvt_f16_f32_e32 v2, v2
; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-FAKE16-NEXT: v_bfi_b32 v0, 0x7fff, v0, s4
+; GFX1200-FAKE16-NEXT: v_bfi_b32 v2, 0x7fff, v2, v0
; GFX1200-FAKE16-NEXT: .LBB0_9: ; %Flow19
-; GFX1200-FAKE16-NEXT: s_cmp_lg_f16 s3, 0
-; GFX1200-FAKE16-NEXT: s_cselect_b32 s3, -1, 0
-; GFX1200-FAKE16-NEXT: s_cmp_nge_f16 s2, 0x7c00
-; GFX1200-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
-; GFX1200-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1200-FAKE16-NEXT: s_and_b32 vcc_lo, s2, s3
+; GFX1200-FAKE16-NEXT: v_dual_mov_b32 v3, 0 :: v_dual_and_b32 v0, 0x7fff, v0
+; GFX1200-FAKE16-NEXT: v_cmp_lg_f16_e32 vcc_lo, 0, v1
+; GFX1200-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1200-FAKE16-NEXT: v_cmp_nle_f16_e64 s0, 0x7c00, v0
+; GFX1200-FAKE16-NEXT: s_and_b32 vcc_lo, s0, vcc_lo
; GFX1200-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX1200-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_cndmask_b32 v0, 0x7e00, v0
-; GFX1200-FAKE16-NEXT: global_store_b16 v1, v0, s[0:1]
+; GFX1200-FAKE16-NEXT: v_cndmask_b32_e32 v0, 0x7e00, v2, vcc_lo
+; GFX1200-FAKE16-NEXT: global_store_b16 v3, v0, s[8:9]
; GFX1200-FAKE16-NEXT: s_endpgm
ptr addrspace(1) %in2) #0 {
%gep2 = getelementptr half, ptr addrspace(1) %in2, i32 4
@@ -1641,35 +1636,34 @@ define amdgpu_kernel void @fast_frem_f16(ptr addrspace(1) %out, ptr addrspace(1)
; GFX1200-TRUE16-NEXT: s_clause 0x1
; GFX1200-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1200-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1200-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: global_load_u16 v3, v2, s[2:3]
-; GFX1200-TRUE16-NEXT: global_load_u16 v4, v2, s[4:5] offset:8
+; GFX1200-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
+; GFX1200-TRUE16-NEXT: global_load_d16_b16 v3, v1, s[4:5] offset:8
; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v4.l
+; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v3.l
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v0, v0
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v1, v3, v0, neg(0) op_sel_hi:[1,0,0]
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v4, v2, v0, neg(0) op_sel_hi:[1,0,0]
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v4, v1, v3 op_sel_hi:[1,0,1]
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v1, v5, v0
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v4, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v4, v1, v3 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1200-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v4.l, v1.l
+; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v3.l, v0.l, v4.l
-; GFX1200-TRUE16-NEXT: global_store_b16 v2, v3, s[0:1]
+; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v3.l
+; GFX1200-TRUE16-NEXT: global_store_b16 v1, v2, s[0:1]
; GFX1200-TRUE16-NEXT: s_endpgm
;
; GFX1200-FAKE16-LABEL: fast_frem_f16:
@@ -2125,35 +2119,34 @@ define amdgpu_kernel void @unsafe_frem_f16(ptr addrspace(1) %out, ptr addrspace(
; GFX1200-TRUE16-NEXT: s_clause 0x1
; GFX1200-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX1200-TRUE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX1200-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX1200-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX1200-TRUE16-NEXT: s_clause 0x1
-; GFX1200-TRUE16-NEXT: global_load_u16 v3, v2, s[2:3]
-; GFX1200-TRUE16-NEXT: global_load_u16 v4, v2, s[4:5] offset:8
+; GFX1200-TRUE16-NEXT: global_load_d16_b16 v2, v1, s[2:3]
+; GFX1200-TRUE16-NEXT: global_load_d16_b16 v3, v1, s[4:5] offset:8
; GFX1200-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v4.l
+; GFX1200-TRUE16-NEXT: v_cvt_f32_f16_e32 v0, v3.l
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(TRANS32_DEP_1)
; GFX1200-TRUE16-NEXT: v_rcp_f32_e32 v0, v0
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v1, v3, v0, neg(0) op_sel_hi:[1,0,0]
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v4, v2, v0, neg(0) op_sel_hi:[1,0,0]
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v4, v1, v3 op_sel_hi:[1,0,1]
-; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v1, v5, v0
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT: v_fmac_f32_e32 v4, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v4, v1, v3 op_sel_hi:[1,0,1]
+; GFX1200-TRUE16-NEXT: v_fma_mix_f32 v5, -v3, v4, v2 op_sel_hi:[1,0,1]
; GFX1200-TRUE16-NEXT: v_mul_f32_e32 v0, v5, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_and_b32_e32 v0, 0xff800000, v0
-; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v1
-; GFX1200-TRUE16-NEXT: v_mov_b16_e32 v1.l, v3.l
-; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1200-TRUE16-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_cvt_f16_f32_e32 v0.l, v0
-; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v4.l, v1.l
+; GFX1200-TRUE16-NEXT: v_div_fixup_f16 v0.l, v0.l, v3.l, v2.l
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1200-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
; GFX1200-TRUE16-NEXT: v_xor_b32_e32 v0, 0x8000, v0
; GFX1200-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v3.l, v0.l, v4.l
-; GFX1200-TRUE16-NEXT: global_store_b16 v2, v3, s[0:1]
+; GFX1200-TRUE16-NEXT: v_fmac_f16_e32 v2.l, v0.l, v3.l
+; GFX1200-TRUE16-NEXT: global_store_b16 v1, v2, s[0:1]
; GFX1200-TRUE16-NEXT: s_endpgm
;
; GFX1200-FAKE16-LABEL: unsafe_frem_f16:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll
index 14886857fa2f0..9a569fd127d73 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.cos.f16.ll
@@ -102,7 +102,7 @@ define amdgpu_kernel void @cos_f16(ptr addrspace(1) %r, ptr addrspace(1) %a) {
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mul_f16_e32 v0.l, 0.15915494, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll
index 1c035c4709bab..ad690fe234bc7 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.rint.f16.ll
@@ -96,7 +96,7 @@ define amdgpu_kernel void @rint_f16(
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_rndne_f16_e32 v0.l, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll
index 0a5df29922f45..d8f53af61b343 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sin.f16.ll
@@ -102,7 +102,7 @@ define amdgpu_kernel void @sin_f16(ptr addrspace(1) %r, ptr addrspace(1) %a) {
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_u16 v0, v1, s[2:3]
+; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_mul_f16_e32 v0.l, 0.15915494, v0.l
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
index 14945be58876a..30d966eb1b7c2 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.bf16.ll
@@ -23,7 +23,7 @@ define amdgpu_kernel void @sqrt_bf16(ptr addrspace(1) %r, ptr addrspace(1) %a) {
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s8, s2
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s9, s3
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-SDAG-NEXT: buffer_load_u16 v0, off, s[8:11], null
+; GFX12-TRUE16-SDAG-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s5, s1
; GFX12-TRUE16-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-SDAG-NEXT: v_sqrt_bf16_e32 v0.l, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
index c9f4e1993cd73..bcaba060b812f 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.sqrt.f16.ll
@@ -129,7 +129,7 @@ define amdgpu_kernel void @sqrt_f16(
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s8, s2
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s9, s3
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-SDAG-NEXT: buffer_load_u16 v0, off, s[8:11], null
+; GFX12-TRUE16-SDAG-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
; GFX12-TRUE16-SDAG-NEXT: s_mov_b32 s5, s1
; GFX12-TRUE16-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-SDAG-NEXT: v_sqrt_f16_e32 v0.l, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll b/llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll
index 9715cb0e3ee5b..4664a4475d22a 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.trunc.f16.ll
@@ -95,7 +95,7 @@ define amdgpu_kernel void @trunc_f16(
; GFX12-TRUE16-NEXT: s_mov_b32 s8, s2
; GFX12-TRUE16-NEXT: s_mov_b32 s9, s3
; GFX12-TRUE16-NEXT: s_mov_b32 s4, s0
-; GFX12-TRUE16-NEXT: buffer_load_u16 v0, off, s[8:11], null
+; GFX12-TRUE16-NEXT: buffer_load_d16_b16 v0, off, s[8:11], null
; GFX12-TRUE16-NEXT: s_mov_b32 s5, s1
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_trunc_f16_e32 v0.l, v0.l
diff --git a/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll b/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
index b1184b56f4b79..99b77b1517bd2 100644
--- a/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
+++ b/llvm/test/CodeGen/AMDGPU/move-to-valu-pseudo-scalar-trans-f16-true16.ll
@@ -9,11 +9,13 @@ define amdgpu_kernel void @exp_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
- ; CHECK-NEXT: [[V_EXP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_EXP_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_EXP_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[V_EXP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_EXP_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_EXP_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
@@ -30,11 +32,13 @@ define amdgpu_kernel void @log_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
- ; CHECK-NEXT: [[V_LOG_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_LOG_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_LOG_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[V_LOG_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_LOG_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_LOG_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
@@ -51,11 +55,13 @@ define amdgpu_kernel void @rcp_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
- ; CHECK-NEXT: [[V_RCP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RCP_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RCP_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[V_RCP_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RCP_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RCP_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
@@ -72,11 +78,13 @@ define amdgpu_kernel void @rsq_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
- ; CHECK-NEXT: [[V_RSQ_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RSQ_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RSQ_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[V_RSQ_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_RSQ_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_RSQ_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
@@ -93,11 +101,13 @@ define amdgpu_kernel void @sqrt_f16(ptr addrspace(1) %ptr) {
; CHECK-NEXT: [[COPY:%[0-9]+]]:sgpr_64(p4) = COPY $sgpr4_sgpr5
; CHECK-NEXT: [[S_LOAD_DWORDX2_IMM:%[0-9]+]]:sreg_64_xexec_xnull = S_LOAD_DWORDX2_IMM [[COPY]](p4), 36, 0 :: (dereferenceable invariant load (s64) from %ir.ptr.kernarg.offset, align 4, addrspace 4)
; CHECK-NEXT: [[V_MOV_B32_e32_:%[0-9]+]]:vgpr_32 = V_MOV_B32_e32 0, implicit $exec
- ; CHECK-NEXT: [[GLOBAL_LOAD_USHORT_SADDR:%[0-9]+]]:vgpr_32 = GLOBAL_LOAD_USHORT_SADDR [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
- ; CHECK-NEXT: [[V_SQRT_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_SQRT_F16_t16_e64 0, [[GLOBAL_LOAD_USHORT_SADDR]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_:%[0-9]+]]:vgpr_16 = GLOBAL_LOAD_SHORT_D16_SADDR_t16 [[S_LOAD_DWORDX2_IMM]], [[V_MOV_B32_e32_]], 0, 0, implicit $exec :: (volatile "amdgpu-noclobber" load (s16) from %ir.ptr.load, addrspace 1)
; CHECK-NEXT: [[DEF:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
- ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SQRT_F16_t16_e64_]], %subreg.lo16, [[DEF]], %subreg.hi16
- ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE]]
+ ; CHECK-NEXT: [[REG_SEQUENCE:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[GLOBAL_LOAD_SHORT_D16_SADDR_t16_]], %subreg.lo16, [[DEF]], %subreg.hi16
+ ; CHECK-NEXT: [[V_SQRT_F16_t16_e64_:%[0-9]+]]:vgpr_16 = nofpexcept V_SQRT_F16_t16_e64 0, killed [[REG_SEQUENCE]].lo16, 0, 0, 0, implicit $mode, implicit $exec
+ ; CHECK-NEXT: [[DEF1:%[0-9]+]]:vgpr_16 = IMPLICIT_DEF
+ ; CHECK-NEXT: [[REG_SEQUENCE1:%[0-9]+]]:vgpr_32 = REG_SEQUENCE [[V_SQRT_F16_t16_e64_]], %subreg.lo16, [[DEF1]], %subreg.hi16
+ ; CHECK-NEXT: [[COPY1:%[0-9]+]]:vgpr_16 = COPY [[REG_SEQUENCE1]]
; CHECK-NEXT: GLOBAL_STORE_SHORT_SADDR_t16 [[V_MOV_B32_e32_]], killed [[COPY1]], [[S_LOAD_DWORDX2_IMM]], 0, 0, implicit $exec :: (store (s16) into %ir.ptr.load, addrspace 1)
; CHECK-NEXT: S_ENDPGM 0
%val = load volatile half, ptr addrspace(1) %ptr
>From aba4174956047c1f36d271bf52e68141d9788f90 Mon Sep 17 00:00:00 2001
From: John Lu <John.Lu at amd.com>
Date: Mon, 28 Sep 2026 22:00:15 -0500
Subject: [PATCH 6/6] Remove TODO
Signed-off-by: John Lu <John.Lu at amd.com>
---
llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp | 9 +-
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 327 +++++++-----------
2 files changed, 137 insertions(+), 199 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
index 40e87a701c5f3..30d899f2cc494 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPUISelLowering.cpp
@@ -846,7 +846,14 @@ bool AMDGPUTargetLowering::shouldReduceLoadWidth(
unsigned AS = MN->getAddressSpace();
// Do not shrink an aligned scalar load to sub-dword.
// Scalar engine cannot do sub-dword loads.
- // TODO: Update this for GFX12 which does have scalar sub-dword loads.
+ // Do not enable for gfx1250+ even though it has sub-dword loads because
+ // this will convert:
+ // i16 = trunc (zextload i16->i32)
+ // to:
+ // i16 = (load i16)
+ // This transformation will be reversed by LowerLOAD resulting in an infinite
+ // loop. Also, tablegen already has a pattern to match zextload i16->i32, but
+ // load i16 will not be matched since there is no instruction that does it.
if (OldSize >= 32 && NewSize < 32 && MN->getAlign() >= Align(4) &&
(AS == AMDGPUAS::CONSTANT_ADDRESS ||
AS == AMDGPUAS::CONSTANT_ADDRESS_32BIT ||
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index e95e1763ce30a..621d0dedb463f 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -7649,150 +7649,74 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; EG-NEXT: MOV T20.W, T20.Z,
; EG-NEXT: MOV * T14.W, T14.Z,
; EG-NEXT: 28(3.923636e-44), 0(0.000000e+00)
-
-; GFX12-TRUE16-LABEL: constant_sextload_v16i1_to_v16i64:
-; GFX12-TRUE16: ; %bb.0:
-; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v32, 0
-; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v32, s[2:3]
-; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-TRUE16-NEXT: v_readfirstlane_b32 s3, v0
-; GFX12-TRUE16-NEXT: s_lshr_b32 s4, s3, 14
-; GFX12-TRUE16-NEXT: s_lshr_b32 s6, s3, 15
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
-; GFX12-TRUE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v28, s3 :: v_dual_mov_b32 v1, s5
-; GFX12-TRUE16-NEXT: s_lshr_b32 s10, s3, 13
-; GFX12-TRUE16-NEXT: s_lshr_b32 s12, s3, 10
-; GFX12-TRUE16-NEXT: s_lshr_b32 s14, s3, 11
-; GFX12-TRUE16-NEXT: s_lshr_b32 s8, s3, 12
-; GFX12-TRUE16-NEXT: s_lshr_b32 s16, s3, 8
-; GFX12-TRUE16-NEXT: s_lshr_b32 s18, s3, 9
-; GFX12-TRUE16-NEXT: s_lshr_b32 s20, s3, 6
-; GFX12-TRUE16-NEXT: s_lshr_b32 s22, s3, 7
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
-; GFX12-TRUE16-NEXT: s_lshr_b32 s24, s3, 4
-; GFX12-TRUE16-NEXT: s_lshr_b32 s26, s3, 5
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
-; GFX12-TRUE16-NEXT: s_lshr_b32 s28, s3, 2
-; GFX12-TRUE16-NEXT: s_lshr_b32 s30, s3, 3
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s7
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v5, s9
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v9, s13
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v8, s12 :: v_dual_mov_b32 v11, s15
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v10, s14 :: v_dual_mov_b32 v13, s17
-; GFX12-TRUE16-NEXT: v_bfe_i32 v28, v28, 0, 1
-; GFX12-TRUE16-NEXT: s_lshr_b32 s2, s3, 1
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v7, s11
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v12, s16 :: v_dual_mov_b32 v15, s19
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v14, s18 :: v_dual_mov_b32 v17, s21
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v16, s20 :: v_dual_mov_b32 v19, s23
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v18, s22 :: v_dual_mov_b32 v21, s25
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v20, s24 :: v_dual_mov_b32 v23, s27
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v22, s26 :: v_dual_mov_b32 v25, s29
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v24, s28 :: v_dual_mov_b32 v27, s31
-; GFX12-TRUE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v26, s30 :: v_dual_mov_b32 v31, s3
-; GFX12-TRUE16-NEXT: v_mov_b32_e32 v30, s2
-; GFX12-TRUE16-NEXT: s_clause 0x1
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
-; GFX12-TRUE16-NEXT: v_ashrrev_i32_e32 v29, 31, v28
-; GFX12-TRUE16-NEXT: s_clause 0x5
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
-; GFX12-TRUE16-NEXT: global_store_b128 v32, v[28:31], s[0:1]
-; GFX12-TRUE16-NEXT: s_endpgm
-
-; GFX12-FAKE16-LABEL: constant_sextload_v16i1_to_v16i64:
-; GFX12-FAKE16: ; %bb.0:
-; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v32, 0
-; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
-; GFX12-FAKE16-NEXT: global_load_u16 v0, v32, s[2:3]
-; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX12-FAKE16-NEXT: v_readfirstlane_b32 s3, v0
-; GFX12-FAKE16-NEXT: s_lshr_b32 s4, s3, 14
-; GFX12-FAKE16-NEXT: s_lshr_b32 s6, s3, 15
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
-; GFX12-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v28, s3 :: v_dual_mov_b32 v1, s5
-; GFX12-FAKE16-NEXT: s_lshr_b32 s10, s3, 13
-; GFX12-FAKE16-NEXT: s_lshr_b32 s12, s3, 10
-; GFX12-FAKE16-NEXT: s_lshr_b32 s14, s3, 11
-; GFX12-FAKE16-NEXT: s_lshr_b32 s8, s3, 12
-; GFX12-FAKE16-NEXT: s_lshr_b32 s16, s3, 8
-; GFX12-FAKE16-NEXT: s_lshr_b32 s18, s3, 9
-; GFX12-FAKE16-NEXT: s_lshr_b32 s20, s3, 6
-; GFX12-FAKE16-NEXT: s_lshr_b32 s22, s3, 7
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
-; GFX12-FAKE16-NEXT: s_lshr_b32 s24, s3, 4
-; GFX12-FAKE16-NEXT: s_lshr_b32 s26, s3, 5
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
-; GFX12-FAKE16-NEXT: s_lshr_b32 s28, s3, 2
-; GFX12-FAKE16-NEXT: s_lshr_b32 s30, s3, 3
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s7
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v5, s9
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v9, s13
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v8, s12 :: v_dual_mov_b32 v11, s15
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v10, s14 :: v_dual_mov_b32 v13, s17
-; GFX12-FAKE16-NEXT: v_bfe_i32 v28, v28, 0, 1
-; GFX12-FAKE16-NEXT: s_lshr_b32 s2, s3, 1
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v7, s11
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v12, s16 :: v_dual_mov_b32 v15, s19
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v14, s18 :: v_dual_mov_b32 v17, s21
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v16, s20 :: v_dual_mov_b32 v19, s23
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v18, s22 :: v_dual_mov_b32 v21, s25
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v20, s24 :: v_dual_mov_b32 v23, s27
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v22, s26 :: v_dual_mov_b32 v25, s29
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v24, s28 :: v_dual_mov_b32 v27, s31
-; GFX12-FAKE16-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v26, s30 :: v_dual_mov_b32 v31, s3
-; GFX12-FAKE16-NEXT: v_mov_b32_e32 v30, s2
-; GFX12-FAKE16-NEXT: s_clause 0x1
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
-; GFX12-FAKE16-NEXT: v_ashrrev_i32_e32 v29, 31, v28
-; GFX12-FAKE16-NEXT: s_clause 0x5
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
-; GFX12-FAKE16-NEXT: global_store_b128 v32, v[28:31], s[0:1]
-; GFX12-FAKE16-NEXT: s_endpgm
-
-
+;
+; GFX12-LABEL: constant_sextload_v16i1_to_v16i64:
+; GFX12: ; %bb.0:
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_lshr_b32 s6, s2, 14
+; GFX12-NEXT: s_lshr_b32 s8, s2, 15
+; GFX12-NEXT: s_lshr_b32 s10, s2, 12
+; GFX12-NEXT: s_lshr_b32 s12, s2, 13
+; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX12-NEXT: s_lshr_b32 s14, s2, 10
+; GFX12-NEXT: s_lshr_b32 s16, s2, 11
+; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v1, s7
+; GFX12-NEXT: s_lshr_b32 s18, s2, 8
+; GFX12-NEXT: s_lshr_b32 s20, s2, 9
+; GFX12-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v3, s9
+; GFX12-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v5, s11
+; GFX12-NEXT: s_lshr_b32 s22, s2, 6
+; GFX12-NEXT: s_lshr_b32 s24, s2, 7
+; GFX12-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v4, s10 :: v_dual_mov_b32 v7, s13
+; GFX12-NEXT: v_dual_mov_b32 v6, s12 :: v_dual_mov_b32 v9, s15
+; GFX12-NEXT: s_lshr_b32 s26, s2, 4
+; GFX12-NEXT: s_lshr_b32 s28, s2, 5
+; GFX12-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
+; GFX12-NEXT: s_lshr_b32 s30, s2, 2
+; GFX12-NEXT: s_lshr_b32 s34, s2, 3
+; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v8, s14 :: v_dual_mov_b32 v11, s17
+; GFX12-NEXT: v_dual_mov_b32 v10, s16 :: v_dual_mov_b32 v13, s19
+; GFX12-NEXT: s_lshr_b32 s36, s2, 1
+; GFX12-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v12, s18 :: v_dual_mov_b32 v15, s21
+; GFX12-NEXT: v_mov_b32_e32 v14, s20
+; GFX12-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:112
+; GFX12-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:96
+; GFX12-NEXT: v_dual_mov_b32 v0, s22 :: v_dual_mov_b32 v3, s25
+; GFX12-NEXT: v_dual_mov_b32 v1, s23 :: v_dual_mov_b32 v2, s24
+; GFX12-NEXT: v_mov_b32_e32 v5, s27
+; GFX12-NEXT: s_bfe_i64 s[4:5], s[2:3], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[2:3], s[36:37], 0x10000
+; GFX12-NEXT: v_dual_mov_b32 v4, s26 :: v_dual_mov_b32 v7, s29
+; GFX12-NEXT: v_dual_mov_b32 v6, s28 :: v_dual_mov_b32 v17, s31
+; GFX12-NEXT: v_dual_mov_b32 v16, s30 :: v_dual_mov_b32 v19, s35
+; GFX12-NEXT: v_dual_mov_b32 v18, s34 :: v_dual_mov_b32 v21, s5
+; GFX12-NEXT: v_dual_mov_b32 v20, s4 :: v_dual_mov_b32 v23, s3
+; GFX12-NEXT: v_mov_b32_e32 v22, s2
+; GFX12-NEXT: s_clause 0x5
+; GFX12-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:80
+; GFX12-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:64
+; GFX12-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:48
+; GFX12-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:32
+; GFX12-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:16
+; GFX12-NEXT: global_store_b128 v24, v[20:23], s[0:1]
+; GFX12-NEXT: s_endpgm
+;
; GFX1250-LABEL: constant_sextload_v16i1_to_v16i64:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -7801,67 +7725,74 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-NEXT: global_prefetch_b8 v0, s[64:65] scope:SCOPE_SE
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: global_load_u16 v0, v32, s[2:3] nv
-; GFX1250-NEXT: s_wait_loadcnt 0x0
-; GFX1250-NEXT: v_readfirstlane_b32 s3, v0
-; GFX1250-NEXT: s_lshr_b32 s4, s3, 14
-; GFX1250-NEXT: s_lshr_b32 s6, s3, 15
-; GFX1250-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
-; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-NEXT: v_dual_mov_b32 v28, s3 :: v_dual_mov_b32 v0, s4
-; GFX1250-NEXT: s_lshr_b32 s10, s3, 13
-; GFX1250-NEXT: s_lshr_b32 s12, s3, 10
-; GFX1250-NEXT: s_lshr_b32 s14, s3, 11
-; GFX1250-NEXT: s_lshr_b32 s8, s3, 12
-; GFX1250-NEXT: s_lshr_b32 s16, s3, 8
-; GFX1250-NEXT: s_lshr_b32 s18, s3, 9
-; GFX1250-NEXT: s_lshr_b32 s20, s3, 6
-; GFX1250-NEXT: s_lshr_b32 s22, s3, 7
-; GFX1250-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
+; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_lshr_b32 s6, s2, 14
+; GFX1250-NEXT: s_lshr_b32 s8, s2, 15
+; GFX1250-NEXT: s_lshr_b32 s10, s2, 12
+; GFX1250-NEXT: s_lshr_b32 s12, s2, 13
+; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
+; GFX1250-NEXT: s_lshr_b32 s14, s2, 10
+; GFX1250-NEXT: s_lshr_b32 s16, s2, 11
; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
-; GFX1250-NEXT: s_lshr_b32 s24, s3, 4
-; GFX1250-NEXT: s_lshr_b32 s26, s3, 5
-; GFX1250-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v24, 0 :: v_dual_mov_b32 v0, s6
+; GFX1250-NEXT: s_lshr_b32 s18, s2, 8
+; GFX1250-NEXT: s_lshr_b32 s20, s2, 9
+; GFX1250-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_mov_b32 v2, s8
+; GFX1250-NEXT: v_dual_mov_b32 v3, s9 :: v_dual_mov_b32 v4, s10
+; GFX1250-NEXT: s_lshr_b32 s22, s2, 6
+; GFX1250-NEXT: s_lshr_b32 s24, s2, 7
; GFX1250-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
-; GFX1250-NEXT: s_lshr_b32 s28, s3, 2
-; GFX1250-NEXT: s_lshr_b32 s30, s3, 3
-; GFX1250-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v5, s11 :: v_dual_mov_b32 v6, s12
+; GFX1250-NEXT: v_dual_mov_b32 v7, s13 :: v_dual_mov_b32 v8, s14
+; GFX1250-NEXT: s_lshr_b32 s26, s2, 4
+; GFX1250-NEXT: s_lshr_b32 s28, s2, 5
; GFX1250-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v2, s6
-; GFX1250-NEXT: v_dual_mov_b32 v3, s7 :: v_dual_mov_b32 v4, s8
-; GFX1250-NEXT: v_dual_mov_b32 v7, s11 :: v_dual_mov_b32 v8, s12
-; GFX1250-NEXT: v_dual_mov_b32 v9, s13 :: v_dual_mov_b32 v10, s14
-; GFX1250-NEXT: v_dual_mov_b32 v11, s15 :: v_dual_mov_b32 v12, s16
-; GFX1250-NEXT: v_bfe_i32 v28, v28, 0, 1
-; GFX1250-NEXT: s_lshr_b32 s2, s3, 1
-; GFX1250-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
+; GFX1250-NEXT: s_lshr_b32 s30, s2, 2
+; GFX1250-NEXT: s_lshr_b32 s34, s2, 3
; GFX1250-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v5, s9 :: v_dual_mov_b32 v6, s10
-; GFX1250-NEXT: v_dual_mov_b32 v13, s17 :: v_dual_mov_b32 v14, s18
-; GFX1250-NEXT: v_dual_mov_b32 v15, s19 :: v_dual_mov_b32 v16, s20
-; GFX1250-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v9, s15 :: v_dual_mov_b32 v10, s16
+; GFX1250-NEXT: v_dual_mov_b32 v11, s17 :: v_dual_mov_b32 v12, s18
+; GFX1250-NEXT: s_lshr_b32 s36, s2, 1
; GFX1250-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v17, s21 :: v_dual_mov_b32 v18, s22
-; GFX1250-NEXT: v_dual_mov_b32 v19, s23 :: v_dual_mov_b32 v20, s24
-; GFX1250-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v21, s25 :: v_dual_mov_b32 v22, s26
-; GFX1250-NEXT: v_dual_mov_b32 v23, s27 :: v_dual_mov_b32 v24, s28
-; GFX1250-NEXT: v_dual_mov_b32 v25, s29 :: v_dual_mov_b32 v26, s30
-; GFX1250-NEXT: v_dual_mov_b32 v27, s31 :: v_dual_mov_b32 v30, s2
-; GFX1250-NEXT: v_dual_mov_b32 v31, s3 :: v_dual_ashrrev_i32 v29, 31, v28
-; GFX1250-NEXT: s_clause 0x7
-; GFX1250-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
-; GFX1250-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
-; GFX1250-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
-; GFX1250-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
-; GFX1250-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
-; GFX1250-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:32
-; GFX1250-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:16
-; GFX1250-NEXT: global_store_b128 v32, v[28:31], s[0:1]
+; GFX1250-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v13, s19 :: v_dual_mov_b32 v14, s20
+; GFX1250-NEXT: v_mov_b32_e32 v15, s21
+; GFX1250-NEXT: s_bfe_i64 s[34:35], s[34:35], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
+; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:112
+; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:96
+; GFX1250-NEXT: s_wait_xcnt 0x1
+; GFX1250-NEXT: v_dual_mov_b32 v0, s22 :: v_dual_mov_b32 v1, s23
+; GFX1250-NEXT: v_dual_mov_b32 v2, s24 :: v_dual_mov_b32 v3, s25
+; GFX1250-NEXT: s_wait_xcnt 0x0
+; GFX1250-NEXT: v_mov_b32_e32 v4, s26
+; GFX1250-NEXT: s_bfe_i64 s[4:5], s[2:3], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[2:3], s[36:37], 0x10000
+; GFX1250-NEXT: v_dual_mov_b32 v5, s27 :: v_dual_mov_b32 v6, s28
+; GFX1250-NEXT: v_dual_mov_b32 v7, s29 :: v_dual_mov_b32 v16, s30
+; GFX1250-NEXT: v_dual_mov_b32 v17, s31 :: v_dual_mov_b32 v18, s34
+; GFX1250-NEXT: v_dual_mov_b32 v19, s35 :: v_dual_mov_b32 v20, s4
+; GFX1250-NEXT: v_dual_mov_b32 v21, s5 :: v_dual_mov_b32 v22, s2
+; GFX1250-NEXT: v_mov_b32_e32 v23, s3
+; GFX1250-NEXT: s_clause 0x5
+; GFX1250-NEXT: global_store_b128 v24, v[8:11], s[0:1] offset:80
+; GFX1250-NEXT: global_store_b128 v24, v[12:15], s[0:1] offset:64
+; GFX1250-NEXT: global_store_b128 v24, v[0:3], s[0:1] offset:48
+; GFX1250-NEXT: global_store_b128 v24, v[4:7], s[0:1] offset:32
+; GFX1250-NEXT: global_store_b128 v24, v[16:19], s[0:1] offset:16
+; GFX1250-NEXT: global_store_b128 v24, v[20:23], s[0:1]
; GFX1250-NEXT: s_endpgm
+
+
+
+
%load = load <16 x i1>, ptr addrspace(4) %in
%ext = sext <16 x i1> %load to <16 x i64>
store <16 x i64> %ext, ptr addrspace(1) %out
More information about the llvm-commits
mailing list