[llvm] [AMDGPU][True16] Legalize extloads into 16-bit registers (PR #195289)
Domenic Nutile via llvm-commits
llvm-commits at lists.llvm.org
Fri May 1 09:46:37 PDT 2026
https://github.com/saxlungs created https://github.com/llvm/llvm-project/pull/195289
None
>From 0bb931bf074f7892a9915a980df0976d3d3d0408 Mon Sep 17 00:00:00 2001
From: Domenic Nutile <domenic.nutile at gmail.com>
Date: Fri, 1 May 2026 12:36:31 -0400
Subject: [PATCH] [AMDGPU][True16] Legalize extloads into 16-bit registers
Still need to update a couple of tests but that can't be done until
!176963 is merged
Signed-off-by: Domenic Nutile <domenic.nutile at gmail.com>
---
.../lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp | 4 +-
.../CodeGen/AMDGPU/GlobalISel/flat-scratch.ll | 182 +++++++++---------
.../AMDGPU/GlobalISel/store-local.128.ll | 64 +++---
.../AMDGPU/GlobalISel/store-local.96.ll | 48 +++--
llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll | 118 ++++++++----
5 files changed, 238 insertions(+), 178 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
index 5fc7e22b5feee..7fea9bec6ea3d 100644
--- a/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/AMDGPULegalizerInfo.cpp
@@ -464,8 +464,8 @@ static bool isLoadStoreSizeLegal(const GCNSubtarget &ST,
MemSize = std::max(MemSize, Align);
#endif
- // Only 1-byte and 2-byte to 32-bit extloads are valid.
- if (MemSize != RegSize && RegSize != 32)
+ // Only allow extloads to up to 32 bits.
+ if (MemSize != RegSize && RegSize > 32)
return false;
if (MemSize > maxSizeForAddrSpace(ST, AS, IsLoad,
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
index a9876ce5d99bc..56bb899b7dde6 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/flat-scratch.ll
@@ -1,15 +1,15 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=+enable-flat-scratch < %s | FileCheck -check-prefix=GFX9 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=+enable-flat-scratch < %s | FileCheck -check-prefix=GFX10 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds < %s | FileCheck -check-prefix=GFX942 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds < %s | FileCheck -check-prefix=GFX11 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds < %s | FileCheck -check-prefix=GFX12 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=-promote-alloca -mattr=+enable-flat-scratch < %s | FileCheck -check-prefix=GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=-promote-alloca -mattr=+enable-flat-scratch < %s | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=-promote-alloca < %s | FileCheck -check-prefix=GFX942 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=-promote-alloca < %s | FileCheck -check-prefix=GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=+real-true16,-promote-alloca < %s | FileCheck -check-prefix=GFX12 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -global-isel -new-reg-bank-select -mattr=-unaligned-access-mode -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=+enable-flat-scratch < %s | FileCheck -check-prefixes=UNALIGNED_GFX9 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -global-isel -new-reg-bank-select -mattr=-unaligned-access-mode -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=+enable-flat-scratch < %s | FileCheck -check-prefixes=UNALIGNED_GFX10 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -global-isel -new-reg-bank-select -mattr=-unaligned-access-mode -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds < %s | FileCheck -check-prefixes=UNALIGNED_GFX942 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -global-isel -new-reg-bank-select -mattr=-unaligned-access-mode -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds < %s | FileCheck -check-prefixes=UNALIGNED_GFX11 %s
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -global-isel -new-reg-bank-select -mattr=-unaligned-access-mode -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds < %s | FileCheck -check-prefixes=UNALIGNED_GFX12 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx900 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=-unaligned-access-mode -mattr=-promote-alloca -mattr=+enable-flat-scratch < %s | FileCheck -check-prefixes=UNALIGNED_GFX9 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1030 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=-unaligned-access-mode -mattr=-promote-alloca -mattr=+enable-flat-scratch < %s | FileCheck -check-prefixes=UNALIGNED_GFX10 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=-unaligned-access-mode -mattr=-promote-alloca < %s | FileCheck -check-prefixes=UNALIGNED_GFX942 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1100 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=-unaligned-access-mode -mattr=-promote-alloca < %s | FileCheck -check-prefixes=UNALIGNED_GFX11 %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 -global-isel -new-reg-bank-select -disable-promote-alloca-to-vector -disable-promote-alloca-to-lds -mattr=+real-true16,-unaligned-access-mode -mattr=-promote-alloca < %s | FileCheck -check-prefixes=UNALIGNED_GFX12 %s
define amdgpu_kernel void @store_load_sindex_kernel(i32 %idx) {
; GFX9-LABEL: store_load_sindex_kernel:
@@ -2689,22 +2689,23 @@ define void @store_load_i64_unaligned(ptr addrspace(5) nocapture %arg) {
; UNALIGNED_GFX11-LABEL: store_load_i64_unaligned:
; UNALIGNED_GFX11: ; %bb.0: ; %bb
; UNALIGNED_GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX11-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_mov_b32 v2, 0
+; UNALIGNED_GFX11-NEXT: v_mov_b16_e32 v1.l, 15
+; UNALIGNED_GFX11-NEXT: v_mov_b16_e32 v1.h, 0
; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v1, off dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:1 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:1 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:2 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:2 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:3 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:3 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:4 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:4 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:5 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:5 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:6 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:6 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:7 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:7 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; UNALIGNED_GFX11-NEXT: scratch_load_u8 v1, v0, off glc dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -2731,23 +2732,24 @@ define void @store_load_i64_unaligned(ptr addrspace(5) nocapture %arg) {
; UNALIGNED_GFX12-NEXT: s_wait_samplecnt 0x0
; UNALIGNED_GFX12-NEXT: s_wait_bvhcnt 0x0
; UNALIGNED_GFX12-NEXT: s_wait_kmcnt 0x0
-; UNALIGNED_GFX12-NEXT: v_dual_mov_b32 v1, 15 :: v_dual_mov_b32 v2, 0
+; UNALIGNED_GFX12-NEXT: v_mov_b16_e32 v1.l, 15
+; UNALIGNED_GFX12-NEXT: v_mov_b16_e32 v1.h, 0
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v1, off scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:1 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:1 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:2 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:2 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:3 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:3 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:4 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:4 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:5 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:5 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:6 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:6 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:7 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:7 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
; UNALIGNED_GFX12-NEXT: scratch_load_u8 v1, v0, off scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3028,32 +3030,33 @@ define void @store_load_v3i32_unaligned(ptr addrspace(5) nocapture %arg) {
; UNALIGNED_GFX11-LABEL: store_load_v3i32_unaligned:
; UNALIGNED_GFX11: ; %bb.0: ; %bb
; UNALIGNED_GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX11-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 0
-; UNALIGNED_GFX11-NEXT: v_mov_b32_e32 v3, 2
+; UNALIGNED_GFX11-NEXT: v_mov_b16_e32 v1.l, 1
+; UNALIGNED_GFX11-NEXT: v_mov_b16_e32 v1.h, 0
+; UNALIGNED_GFX11-NEXT: v_mov_b16_e32 v2.l, 2
; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v1, off dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:1 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:1 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:2 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:2 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:3 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:3 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: v_mov_b32_e32 v1, 3
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v3, off offset:4 dlc
+; UNALIGNED_GFX11-NEXT: v_mov_b16_e32 v1.l, 3
+; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:4 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:5 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:5 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:6 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:6 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:7 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:7 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v1, off offset:8 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:9 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:9 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:10 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:10 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:11 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:11 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; UNALIGNED_GFX11-NEXT: scratch_load_u8 v1, v0, off glc dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3088,33 +3091,34 @@ define void @store_load_v3i32_unaligned(ptr addrspace(5) nocapture %arg) {
; UNALIGNED_GFX12-NEXT: s_wait_samplecnt 0x0
; UNALIGNED_GFX12-NEXT: s_wait_bvhcnt 0x0
; UNALIGNED_GFX12-NEXT: s_wait_kmcnt 0x0
-; UNALIGNED_GFX12-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 0
-; UNALIGNED_GFX12-NEXT: v_mov_b32_e32 v3, 2
+; UNALIGNED_GFX12-NEXT: v_mov_b16_e32 v1.l, 1
+; UNALIGNED_GFX12-NEXT: v_mov_b16_e32 v1.h, 0
+; UNALIGNED_GFX12-NEXT: v_mov_b16_e32 v2.l, 2
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v1, off scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:1 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:1 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:2 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:2 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:3 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:3 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: v_mov_b32_e32 v1, 3
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v3, off offset:4 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: v_mov_b16_e32 v1.l, 3
+; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:4 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:5 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:5 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:6 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:6 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:7 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:7 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v1, off offset:8 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:9 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:9 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:10 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:10 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:11 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:11 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
; UNALIGNED_GFX12-NEXT: scratch_load_u8 v1, v0, off scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_loadcnt 0x0
@@ -3458,41 +3462,42 @@ define void @store_load_v4i32_unaligned(ptr addrspace(5) nocapture %arg) {
; UNALIGNED_GFX11-LABEL: store_load_v4i32_unaligned:
; UNALIGNED_GFX11: ; %bb.0: ; %bb
; UNALIGNED_GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; UNALIGNED_GFX11-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 0
-; UNALIGNED_GFX11-NEXT: v_mov_b32_e32 v3, 2
+; UNALIGNED_GFX11-NEXT: v_mov_b16_e32 v1.l, 1
+; UNALIGNED_GFX11-NEXT: v_mov_b16_e32 v1.h, 0
+; UNALIGNED_GFX11-NEXT: v_mov_b16_e32 v2.l, 2
; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v1, off dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:1 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:1 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:2 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:2 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:3 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:3 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v3, off offset:4 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:4 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: v_mov_b32_e32 v1, 3
-; UNALIGNED_GFX11-NEXT: v_mov_b32_e32 v3, 4
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:5 dlc
+; UNALIGNED_GFX11-NEXT: v_mov_b16_e32 v1.l, 3
+; UNALIGNED_GFX11-NEXT: v_mov_b16_e32 v2.l, 4
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:5 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:6 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:6 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:7 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:7 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v1, off offset:8 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:9 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:9 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:10 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:10 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:11 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:11 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v3, off offset:12 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:12 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:13 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:13 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:14 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:14 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
-; UNALIGNED_GFX11-NEXT: scratch_store_b8 v0, v2, off offset:15 dlc
+; UNALIGNED_GFX11-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:15 dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt_vscnt null, 0x0
; UNALIGNED_GFX11-NEXT: scratch_load_u8 v1, v0, off glc dlc
; UNALIGNED_GFX11-NEXT: s_waitcnt vmcnt(0)
@@ -3535,42 +3540,43 @@ define void @store_load_v4i32_unaligned(ptr addrspace(5) nocapture %arg) {
; UNALIGNED_GFX12-NEXT: s_wait_samplecnt 0x0
; UNALIGNED_GFX12-NEXT: s_wait_bvhcnt 0x0
; UNALIGNED_GFX12-NEXT: s_wait_kmcnt 0x0
-; UNALIGNED_GFX12-NEXT: v_dual_mov_b32 v1, 1 :: v_dual_mov_b32 v2, 0
-; UNALIGNED_GFX12-NEXT: v_mov_b32_e32 v3, 2
+; UNALIGNED_GFX12-NEXT: v_mov_b16_e32 v1.l, 1
+; UNALIGNED_GFX12-NEXT: v_mov_b16_e32 v1.h, 0
+; UNALIGNED_GFX12-NEXT: v_mov_b16_e32 v2.l, 2
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v1, off scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:1 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:1 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:2 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:2 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:3 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:3 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v3, off offset:4 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:4 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: v_mov_b32_e32 v1, 3
-; UNALIGNED_GFX12-NEXT: v_mov_b32_e32 v3, 4
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:5 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: v_mov_b16_e32 v1.l, 3
+; UNALIGNED_GFX12-NEXT: v_mov_b16_e32 v2.l, 4
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:5 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:6 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:6 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:7 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:7 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v1, off offset:8 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:9 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:9 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:10 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:10 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:11 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:11 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v3, off offset:12 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:12 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:13 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:13 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:14 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:14 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
-; UNALIGNED_GFX12-NEXT: scratch_store_b8 v0, v2, off offset:15 scope:SCOPE_SYS
+; UNALIGNED_GFX12-NEXT: scratch_store_d16_hi_b8 v0, v1, off offset:15 scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_storecnt 0x0
; UNALIGNED_GFX12-NEXT: scratch_load_u8 v1, v0, off scope:SCOPE_SYS
; UNALIGNED_GFX12-NEXT: s_wait_loadcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
index 10e83b70a57d4..ec4f3efa2d150 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
@@ -239,47 +239,53 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_and_b32 s6, 0xffff, s0
; GFX11-NEXT: s_lshr_b32 s5, s0, 16
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
-; GFX11-NEXT: s_lshr_b32 s4, s1, 16
; GFX11-NEXT: s_and_b32 s7, 0xffff, s1
; GFX11-NEXT: s_lshr_b32 s6, s6, 8
-; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, 24
-; GFX11-NEXT: v_dual_mov_b32 v4, s5 :: v_dual_mov_b32 v5, s0
+; GFX11-NEXT: v_mov_b32_e32 v5, s4
+; GFX11-NEXT: v_mov_b16_e32 v2.l, s0
; GFX11-NEXT: s_lshr_b32 s0, s7, 8
-; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s6
+; GFX11-NEXT: v_mov_b16_e32 v3.l, s6
+; GFX11-NEXT: v_mov_b16_e32 v1.h, s5
+; GFX11-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-NEXT: v_mov_b16_e32 v0.h, s1
+; GFX11-NEXT: s_lshr_b32 s1, s1, 24
; GFX11-NEXT: s_and_b32 s9, 0xffff, s2
-; GFX11-NEXT: v_dual_mov_b32 v8, s1 :: v_dual_mov_b32 v9, s0
+; GFX11-NEXT: v_mov_b16_e32 v4.l, s0
; GFX11-NEXT: s_lshr_b32 s0, s2, 24
+; GFX11-NEXT: v_mov_b16_e32 v2.h, s4
+; GFX11-NEXT: v_mov_b16_e32 v3.h, s1
; GFX11-NEXT: s_lshr_b32 s1, s9, 8
-; GFX11-NEXT: ds_store_b8 v1, v0
-; GFX11-NEXT: ds_store_b8 v1, v7 offset:1
-; GFX11-NEXT: ds_store_b8 v1, v4 offset:2
-; GFX11-NEXT: ds_store_b8 v1, v5 offset:3
-; GFX11-NEXT: ds_store_b8 v1, v2 offset:4
-; GFX11-NEXT: ds_store_b8 v1, v9 offset:5
-; GFX11-NEXT: ds_store_b8 v1, v6 offset:6
-; GFX11-NEXT: ds_store_b8 v1, v8 offset:7
-; GFX11-NEXT: v_mov_b32_e32 v4, s0
+; GFX11-NEXT: ds_store_b8 v5, v0
+; GFX11-NEXT: ds_store_b8 v5, v3 offset:1
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v1 offset:2
+; GFX11-NEXT: ds_store_b8 v5, v2 offset:3
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v0 offset:4
+; GFX11-NEXT: ds_store_b8 v5, v4 offset:5
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v2 offset:6
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v3 offset:7
+; GFX11-NEXT: v_mov_b16_e32 v1.h, s0
; GFX11-NEXT: s_and_b32 s0, 0xffff, s3
; GFX11-NEXT: s_lshr_b32 s8, s2, 16
-; GFX11-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v5, s3
+; GFX11-NEXT: v_mov_b16_e32 v1.l, s2
+; GFX11-NEXT: v_mov_b16_e32 v0.l, s1
; GFX11-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-NEXT: v_mov_b16_e32 v0.h, s8
; GFX11-NEXT: s_lshr_b32 s1, s3, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v7, s1
-; GFX11-NEXT: v_mov_b32_e32 v6, s0
+; GFX11-NEXT: v_mov_b16_e32 v2.h, s0
; GFX11-NEXT: s_lshr_b32 s0, s3, 24
-; GFX11-NEXT: v_mov_b32_e32 v8, s0
-; GFX11-NEXT: ds_store_b8 v1, v3 offset:8
-; GFX11-NEXT: ds_store_b8 v1, v0 offset:9
-; GFX11-NEXT: ds_store_b8 v1, v2 offset:10
-; GFX11-NEXT: ds_store_b8 v1, v4 offset:11
-; GFX11-NEXT: ds_store_b8 v1, v5 offset:12
-; GFX11-NEXT: ds_store_b8 v1, v6 offset:13
-; GFX11-NEXT: ds_store_b8 v1, v7 offset:14
-; GFX11-NEXT: ds_store_b8 v1, v8 offset:15
+; GFX11-NEXT: v_mov_b16_e32 v2.l, s3
+; GFX11-NEXT: v_mov_b16_e32 v3.l, s1
+; GFX11-NEXT: v_mov_b16_e32 v3.h, s0
+; GFX11-NEXT: ds_store_b8 v5, v1 offset:8
+; GFX11-NEXT: ds_store_b8 v5, v0 offset:9
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v0 offset:10
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v1 offset:11
+; GFX11-NEXT: ds_store_b8 v5, v2 offset:12
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v2 offset:13
+; GFX11-NEXT: ds_store_b8 v5, v3 offset:14
+; GFX11-NEXT: ds_store_b8_d16_hi v5, v3 offset:15
; GFX11-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out, align 1
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
index db5d0fdb5c036..158b5d7450609 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
@@ -206,37 +206,43 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX11-NEXT: s_and_b32 s5, 0xffff, s0
-; GFX11-NEXT: s_lshr_b32 s7, s2, 16
-; GFX11-NEXT: s_lshr_b32 s5, s5, 8
; GFX11-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-NEXT: s_lshr_b32 s5, s5, 8
+; GFX11-NEXT: v_mov_b16_e32 v0.l, s0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: v_mov_b32_e32 v6, s3
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
; GFX11-NEXT: s_lshr_b32 s3, s1, 16
; GFX11-NEXT: s_and_b32 s6, 0xffff, s1
-; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT: v_mov_b16_e32 v0.h, s1
; GFX11-NEXT: s_lshr_b32 s1, s1, 24
; GFX11-NEXT: s_and_b32 s8, 0xffff, s2
-; GFX11-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v9, s5
-; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s0
+; GFX11-NEXT: v_mov_b16_e32 v4.l, s5
+; GFX11-NEXT: v_mov_b16_e32 v1.h, s4
+; GFX11-NEXT: s_lshr_b32 s7, s2, 16
+; GFX11-NEXT: v_mov_b16_e32 v1.l, s2
; GFX11-NEXT: s_lshr_b32 s2, s2, 24
+; GFX11-NEXT: v_mov_b16_e32 v2.l, s0
; GFX11-NEXT: s_lshr_b32 s0, s6, 8
-; GFX11-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s1
+; GFX11-NEXT: v_mov_b16_e32 v3.l, s1
; GFX11-NEXT: s_lshr_b32 s1, s8, 8
-; GFX11-NEXT: v_dual_mov_b32 v10, s2 :: v_dual_mov_b32 v11, s0
-; GFX11-NEXT: v_mov_b32_e32 v12, s1
-; GFX11-NEXT: ds_store_b8 v1, v0
-; GFX11-NEXT: ds_store_b8 v1, v9 offset:1
-; GFX11-NEXT: ds_store_b8 v1, v4 offset:2
-; GFX11-NEXT: ds_store_b8 v1, v5 offset:3
-; GFX11-NEXT: ds_store_b8 v1, v2 offset:4
-; GFX11-NEXT: ds_store_b8 v1, v11 offset:5
-; GFX11-NEXT: ds_store_b8 v1, v6 offset:6
-; GFX11-NEXT: ds_store_b8 v1, v7 offset:7
-; GFX11-NEXT: ds_store_b8 v1, v3 offset:8
-; GFX11-NEXT: ds_store_b8 v1, v12 offset:9
-; GFX11-NEXT: ds_store_b8 v1, v8 offset:10
-; GFX11-NEXT: ds_store_b8 v1, v10 offset:11
+; GFX11-NEXT: v_mov_b16_e32 v2.h, s3
+; GFX11-NEXT: v_mov_b16_e32 v3.h, s7
+; GFX11-NEXT: v_mov_b16_e32 v4.h, s2
+; GFX11-NEXT: v_mov_b16_e32 v5.l, s0
+; GFX11-NEXT: v_mov_b16_e32 v5.h, s1
+; GFX11-NEXT: ds_store_b8 v6, v0
+; GFX11-NEXT: ds_store_b8 v6, v4 offset:1
+; GFX11-NEXT: ds_store_b8_d16_hi v6, v1 offset:2
+; GFX11-NEXT: ds_store_b8 v6, v2 offset:3
+; GFX11-NEXT: ds_store_b8_d16_hi v6, v0 offset:4
+; GFX11-NEXT: ds_store_b8 v6, v5 offset:5
+; GFX11-NEXT: ds_store_b8_d16_hi v6, v2 offset:6
+; GFX11-NEXT: ds_store_b8 v6, v3 offset:7
+; GFX11-NEXT: ds_store_b8 v6, v1 offset:8
+; GFX11-NEXT: ds_store_b8_d16_hi v6, v5 offset:9
+; GFX11-NEXT: ds_store_b8_d16_hi v6, v3 offset:10
+; GFX11-NEXT: ds_store_b8_d16_hi v6, v4 offset:11
; GFX11-NEXT: s_endpgm
store <3 x i32> %x, ptr addrspace(3) %out, align 1
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
index a5576343af219..bc1b8faf5ecf2 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-saddr-load.ll
@@ -2331,15 +2331,15 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_i8 v0, v0, s[2:3]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
-; GFX1250-GISEL-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -2349,6 +2349,16 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
+; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -2374,15 +2384,15 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
; GFX1250-SDAG-FAKE16-NEXT: v_bfi_b32 v0, 0xffff, v0, v1
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
-; GFX1250-GISEL-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -2392,6 +2402,16 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128(p
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b16_e32 v0.h, v1.h
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
+; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, 0xffff0000, v1, v0
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16lo_sexti8_reg_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -2802,16 +2822,16 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_i8 v0, v0, s[2:3]
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
-; GFX1250-GISEL-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3]
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -2823,6 +2843,17 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi(ptr inreg %
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
+; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3]
+; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
@@ -2848,16 +2879,16 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
; GFX1250-SDAG-FAKE16-NEXT: v_perm_b32 v0, v0, v1, 0x5040100
; GFX1250-SDAG-FAKE16-NEXT: ; return to shader part epilog
;
-; GFX1250-GISEL-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
-; GFX1250-GISEL: ; %bb.0:
-; GFX1250-GISEL-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-GISEL-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
-; GFX1250-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
-; GFX1250-GISEL-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1250-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX1250-GISEL-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX1250-GISEL-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
-; GFX1250-GISEL-NEXT: ; return to shader part epilog
+; GFX1250-GISEL-FAKE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
+; GFX1250-GISEL-FAKE16: ; %bb.0:
+; GFX1250-GISEL-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-FAKE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1250-GISEL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-FAKE16-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
+; GFX1250-GISEL-FAKE16-NEXT: ; return to shader part epilog
;
; GFX1250-SDAG-TRUE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
; GFX1250-SDAG-TRUE16: ; %bb.0:
@@ -2869,6 +2900,17 @@ define amdgpu_ps <2 x half> @flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128(p
; GFX1250-SDAG-TRUE16-NEXT: v_mov_b32_e32 v0, v1
; GFX1250-SDAG-TRUE16-NEXT: ; return to shader part epilog
;
+; GFX1250-GISEL-TRUE16-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
+; GFX1250-GISEL-TRUE16: ; %bb.0:
+; GFX1250-GISEL-TRUE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-GISEL-TRUE16-NEXT: flat_load_i8 v0, v0, s[2:3] offset:-128
+; GFX1250-GISEL-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
+; GFX1250-GISEL-TRUE16-NEXT: v_mov_b16_e32 v0.h, 0
+; GFX1250-GISEL-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-GISEL-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX1250-GISEL-TRUE16-NEXT: v_and_or_b32 v0, 0xffff, v1, v0
+; GFX1250-GISEL-TRUE16-NEXT: ; return to shader part epilog
+;
; GFX1250-NOECC-LABEL: flat_load_saddr_i16_d16hi_sexti8_reg_hi_immneg128:
; GFX1250-NOECC: ; %bb.0:
; GFX1250-NOECC-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
More information about the llvm-commits
mailing list