[llvm] ace44dc - [AMDGPU] Gate `S_LSHL[1-4]_ADD_U32` patterns on uniform results (#198508)
via llvm-commits
llvm-commits at lists.llvm.org
Tue May 19 10:56:09 PDT 2026
Author: Arseniy Obolenskiy
Date: 2026-05-19T19:56:04+02:00
New Revision: ace44dc5a2a48e5d024bc5c015f69a67cd9be909
URL: https://github.com/llvm/llvm-project/commit/ace44dc5a2a48e5d024bc5c015f69a67cd9be909
DIFF: https://github.com/llvm/llvm-project/commit/ace44dc5a2a48e5d024bc5c015f69a67cd9be909.diff
LOG: [AMDGPU] Gate `S_LSHL[1-4]_ADD_U32` patterns on uniform results (#198508)
Like the other SOP2 patterns in this file, these scalar instructions
require the result to be uniform. Wrap them in `UniformBinFrag` so
divergent shl/add chains use `V_LSHL_ADD_U32`
Added:
Modified:
llvm/lib/Target/AMDGPU/SOPInstructions.td
llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll
llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-preserve-cc.ll
llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
llvm/test/CodeGen/AMDGPU/flat-scratch.ll
llvm/test/CodeGen/AMDGPU/gep-flags-stack-offsets.ll
llvm/test/CodeGen/AMDGPU/hip.extern.shared.array.ll
llvm/test/CodeGen/AMDGPU/lds-relocs.ll
llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
llvm/test/CodeGen/AMDGPU/shl_add.ll
llvm/test/CodeGen/AMDGPU/wqm.ll
Removed:
################################################################################
diff --git a/llvm/lib/Target/AMDGPU/SOPInstructions.td b/llvm/lib/Target/AMDGPU/SOPInstructions.td
index e0bdcf61440f3..b49bd100f9d90 100644
--- a/llvm/lib/Target/AMDGPU/SOPInstructions.td
+++ b/llvm/lib/Target/AMDGPU/SOPInstructions.td
@@ -907,16 +907,16 @@ let SubtargetPredicate = isGFX9Plus in {
let Defs = [SCC] in {
def S_LSHL1_ADD_U32 : SOP2_32<"s_lshl1_add_u32",
- [(set i32:$sdst, (shl1_add SSrc_b32:$src0, SSrc_b32:$src1))]
+ [(set i32:$sdst, (UniformBinFrag<shl1_add> SSrc_b32:$src0, SSrc_b32:$src1))]
>;
def S_LSHL2_ADD_U32 : SOP2_32<"s_lshl2_add_u32",
- [(set i32:$sdst, (shl2_add SSrc_b32:$src0, SSrc_b32:$src1))]
+ [(set i32:$sdst, (UniformBinFrag<shl2_add> SSrc_b32:$src0, SSrc_b32:$src1))]
>;
def S_LSHL3_ADD_U32 : SOP2_32<"s_lshl3_add_u32",
- [(set i32:$sdst, (shl3_add SSrc_b32:$src0, SSrc_b32:$src1))]
+ [(set i32:$sdst, (UniformBinFrag<shl3_add> SSrc_b32:$src0, SSrc_b32:$src1))]
>;
def S_LSHL4_ADD_U32 : SOP2_32<"s_lshl4_add_u32",
- [(set i32:$sdst, (shl4_add SSrc_b32:$src0, SSrc_b32:$src1))]
+ [(set i32:$sdst, (UniformBinFrag<shl4_add> SSrc_b32:$src0, SSrc_b32:$src1))]
>;
} // End Defs = [SCC]
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll
index 89d5d3dc18ea8..b97c5c1e2431e 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-cc.ll
@@ -1149,27 +1149,27 @@ define amdgpu_cs_chain void @amdgpu_cs_chain_realign_stack_chain_call(i32 %idx,
; DAGISEL-GFX10: ; %bb.0:
; DAGISEL-GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; DAGISEL-GFX10-NEXT: s_add_i32 s33, s32, 0x3e0
-; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v2, 3
+; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v0, 4
; DAGISEL-GFX10-NEXT: s_and_b32 s33, s33, 0xfffffc00
; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v3, 2
-; DAGISEL-GFX10-NEXT: v_lshrrev_b32_e64 v1, 5, s33
+; DAGISEL-GFX10-NEXT: v_lshrrev_b32_e64 v2, 5, s33
; DAGISEL-GFX10-NEXT: s_mov_b32 s34, s32
; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v4, 1
; DAGISEL-GFX10-NEXT: s_mov_b32 s5, chain_callee at abs32@hi
; DAGISEL-GFX10-NEXT: s_mov_b32 s4, chain_callee at abs32@lo
-; DAGISEL-GFX10-NEXT: v_lshl_add_u32 v0, v8, 4, v1
-; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v1, 4
+; DAGISEL-GFX10-NEXT: v_lshl_add_u32 v1, v8, 4, v2
; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v8, v9
; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v9, v10
; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v10, v11
+; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v2, 3
; DAGISEL-GFX10-NEXT: s_addk_i32 s32, 0x1800
-; DAGISEL-GFX10-NEXT: buffer_store_dword v1, v0, s[48:51], 0 offen offset:12
+; DAGISEL-GFX10-NEXT: buffer_store_dword v0, v1, s[48:51], 0 offen offset:12
; DAGISEL-GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; DAGISEL-GFX10-NEXT: buffer_store_dword v2, v0, s[48:51], 0 offen offset:8
+; DAGISEL-GFX10-NEXT: buffer_store_dword v2, v1, s[48:51], 0 offen offset:8
; DAGISEL-GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; DAGISEL-GFX10-NEXT: buffer_store_dword v3, v0, s[48:51], 0 offen offset:4
+; DAGISEL-GFX10-NEXT: buffer_store_dword v3, v1, s[48:51], 0 offen offset:4
; DAGISEL-GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; DAGISEL-GFX10-NEXT: buffer_store_dword v4, v0, s[48:51], 0 offen
+; DAGISEL-GFX10-NEXT: buffer_store_dword v4, v1, s[48:51], 0 offen
; DAGISEL-GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; DAGISEL-GFX10-NEXT: s_mov_b32 s32, s34
; DAGISEL-GFX10-NEXT: s_mov_b32 exec_lo, -1
diff --git a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-preserve-cc.ll b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-preserve-cc.ll
index cdd569fa17236..c5602b7c11c7a 100644
--- a/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-preserve-cc.ll
+++ b/llvm/test/CodeGen/AMDGPU/amdgpu-cs-chain-preserve-cc.ll
@@ -688,7 +688,7 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_realign_stack(i32
; DAGISEL-GFX10: ; %bb.0:
; DAGISEL-GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; DAGISEL-GFX10-NEXT: s_add_i32 s33, s32, 0x3e0
-; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v2, 4
+; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v1, 4
; DAGISEL-GFX10-NEXT: s_and_b32 s33, s33, 0xfffffc00
; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v3, 3
; DAGISEL-GFX10-NEXT: v_lshrrev_b32_e64 v0, 5, s33
@@ -701,20 +701,20 @@ define amdgpu_cs_chain_preserve void @amdgpu_cs_chain_preserve_realign_stack(i32
; DAGISEL-GFX10-NEXT: s_mov_b32 s5, chain_preserve_callee at abs32@hi
; DAGISEL-GFX10-NEXT: s_mov_b32 s4, chain_preserve_callee at abs32@lo
; DAGISEL-GFX10-NEXT: s_addk_i32 s32, 0x1c00
-; DAGISEL-GFX10-NEXT: v_lshl_add_u32 v1, v8, 4, v0
+; DAGISEL-GFX10-NEXT: v_lshl_add_u32 v2, v8, 4, v0
; DAGISEL-GFX10-NEXT: ;;#ASMSTART
; DAGISEL-GFX10-NEXT: s_nop
; DAGISEL-GFX10-NEXT: ;;#ASMEND
; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v8, v9
; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v9, v10
; DAGISEL-GFX10-NEXT: v_mov_b32_e32 v10, v11
-; DAGISEL-GFX10-NEXT: buffer_store_dword v2, v1, s[48:51], 0 offen offset:12
+; DAGISEL-GFX10-NEXT: buffer_store_dword v1, v2, s[48:51], 0 offen offset:12
; DAGISEL-GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; DAGISEL-GFX10-NEXT: buffer_store_dword v3, v1, s[48:51], 0 offen offset:8
+; DAGISEL-GFX10-NEXT: buffer_store_dword v3, v2, s[48:51], 0 offen offset:8
; DAGISEL-GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; DAGISEL-GFX10-NEXT: buffer_store_dword v4, v1, s[48:51], 0 offen offset:4
+; DAGISEL-GFX10-NEXT: buffer_store_dword v4, v2, s[48:51], 0 offen offset:4
; DAGISEL-GFX10-NEXT: s_waitcnt_vscnt null, 0x0
-; DAGISEL-GFX10-NEXT: buffer_store_dword v5, v1, s[48:51], 0 offen
+; DAGISEL-GFX10-NEXT: buffer_store_dword v5, v2, s[48:51], 0 offen
; DAGISEL-GFX10-NEXT: s_waitcnt_vscnt null, 0x0
; DAGISEL-GFX10-NEXT: buffer_load_dword v16, off, s[48:51], s33 ; 4-byte Folded Reload
; DAGISEL-GFX10-NEXT: s_mov_b32 s0, s3
diff --git a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
index 95c1dd4d35f86..6c226dea92980 100644
--- a/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
+++ b/llvm/test/CodeGen/AMDGPU/dynamic_stackalloc.ll
@@ -380,13 +380,11 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent() {
define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent_over_aligned() {
; GFX9-SDAG-LABEL: test_dynamic_stackalloc_kernel_divergent_over_aligned:
; GFX9-SDAG: ; %bb.0:
-; GFX9-SDAG-NEXT: s_movk_i32 s32, 0x2000
; GFX9-SDAG-NEXT: s_add_u32 s0, s0, s17
-; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, 0
-; GFX9-SDAG-NEXT: s_add_i32 s4, s32, 0x1fff
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
; GFX9-SDAG-NEXT: s_mov_b32 s33, 0
-; GFX9-SDAG-NEXT: s_and_b32 s6, s4, 0xffffe000
+; GFX9-SDAG-NEXT: s_movk_i32 s32, 0x2000
+; GFX9-SDAG-NEXT: s_addc_u32 s1, s1, 0
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0x1ff0, v0
; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
@@ -402,10 +400,12 @@ define amdgpu_kernel void @test_dynamic_stackalloc_kernel_divergent_over_aligned
; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-SDAG-NEXT: s_nop 1
; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-SDAG-NEXT: v_readlane_b32 s7, v1, 63
+; GFX9-SDAG-NEXT: v_readlane_b32 s6, v1, 63
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v2, s7, 6, v0
+; GFX9-SDAG-NEXT: s_add_i32 s4, s32, 0x1fff
+; GFX9-SDAG-NEXT: s_and_b32 s4, s4, 0xffffe000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v2, s6, 6, v0
; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v2
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x1bc
; GFX9-SDAG-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
@@ -1313,18 +1313,16 @@ define void @test_dynamic_stackalloc_device_uniform_over_aligned(i32 %n) {
; GFX9-SDAG-LABEL: test_dynamic_stackalloc_device_uniform_over_aligned:
; GFX9-SDAG: ; %bb.0:
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_mov_b32 s8, s33
+; GFX9-SDAG-NEXT: s_mov_b32 s7, s33
; GFX9-SDAG-NEXT: s_add_i32 s33, s32, 0x1fc0
; GFX9-SDAG-NEXT: s_and_b32 s33, s33, 0xffffe000
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s33 ; 4-byte Folded Spill
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-SDAG-NEXT: s_mov_b32 s9, s34
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
+; GFX9-SDAG-NEXT: s_mov_b32 s8, s34
; GFX9-SDAG-NEXT: s_mov_b32 s34, s32
; GFX9-SDAG-NEXT: s_addk_i32 s32, 0x4000
-; GFX9-SDAG-NEXT: s_add_i32 s4, s32, 0x1fff
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
-; GFX9-SDAG-NEXT: s_and_b32 s6, s4, 0xffffe000
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v0, s[4:5]
@@ -1340,20 +1338,22 @@ define void @test_dynamic_stackalloc_device_uniform_over_aligned(i32 %n) {
; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-SDAG-NEXT: s_nop 1
; GFX9-SDAG-NEXT: v_max_u32_dpp v1, v1, v1 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-SDAG-NEXT: v_readlane_b32 s7, v1, 63
+; GFX9-SDAG-NEXT: v_readlane_b32 s6, v1, 63
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s6
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v2, s7, 6, v0
+; GFX9-SDAG-NEXT: s_add_i32 s4, s32, 0x1fff
+; GFX9-SDAG-NEXT: s_and_b32 s4, s4, 0xffffe000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s4
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v2, s6, 6, v0
; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v2
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 10
; GFX9-SDAG-NEXT: buffer_store_dword v2, v0, s[0:3], 0 offen
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: s_mov_b32 s32, s34
-; GFX9-SDAG-NEXT: s_mov_b32 s34, s9
+; GFX9-SDAG-NEXT: s_mov_b32 s34, s8
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: buffer_load_dword v1, off, s[0:3], s33 ; 4-byte Folded Reload
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-SDAG-NEXT: s_mov_b32 s33, s8
+; GFX9-SDAG-NEXT: s_mov_b32 s33, s7
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -2271,22 +2271,20 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) {
; GFX9-SDAG-LABEL: test_dynamic_stackalloc_device_multiple_allocas:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_mov_b32 s10, s33
+; GFX9-SDAG-NEXT: s_mov_b32 s9, s33
; GFX9-SDAG-NEXT: s_add_i32 s33, s32, 0xfc0
; GFX9-SDAG-NEXT: s_and_b32 s33, s33, 0xfffff000
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: buffer_store_dword v2, off, s[0:3], s33 offset:64 ; 4-byte Folded Spill
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-SDAG-NEXT: s_mov_b32 s11, s34
+; GFX9-SDAG-NEXT: s_mov_b32 s10, s34
; GFX9-SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v0
; GFX9-SDAG-NEXT: s_mov_b32 s34, s32
; GFX9-SDAG-NEXT: s_addk_i32 s32, 0x3000
; GFX9-SDAG-NEXT: s_and_saveexec_b64 s[4:5], vcc
; GFX9-SDAG-NEXT: s_cbranch_execz .LBB14_2
; GFX9-SDAG-NEXT: ; %bb.1: ; %bb.0
-; GFX9-SDAG-NEXT: s_add_i32 s6, s32, 0xfff
; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, v1, 2, 15
-; GFX9-SDAG-NEXT: s_and_b32 s8, s6, 0xfffff000
; GFX9-SDAG-NEXT: v_and_b32_e32 v1, -16, v1
; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[6:7], -1
; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v1, s[6:7]
@@ -2302,10 +2300,12 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) {
; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-SDAG-NEXT: s_nop 1
; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-SDAG-NEXT: v_readlane_b32 s9, v2, 63
+; GFX9-SDAG-NEXT: v_readlane_b32 s8, v2, 63
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s8
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v3, s9, 6, v1
+; GFX9-SDAG-NEXT: s_add_i32 s6, s32, 0xfff
+; GFX9-SDAG-NEXT: s_and_b32 s6, s6, 0xfffff000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v3, s8, 6, v1
; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v3
; GFX9-SDAG-NEXT: v_and_b32_e32 v3, 0x3ff, v31
; GFX9-SDAG-NEXT: v_lshl_add_u32 v3, v3, 2, 15
@@ -2339,6 +2339,7 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) {
; GFX9-SDAG-NEXT: .LBB14_2: ; %bb.1
; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v0, 2, 15
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 2
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, -16, v0
; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[4:5]
@@ -2363,15 +2364,14 @@ define void @test_dynamic_stackalloc_device_multiple_allocas(i32 %n, i32 %m) {
; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 1
; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s33
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 2
-; GFX9-SDAG-NEXT: buffer_store_dword v0, off, s[0:3], s4
+; GFX9-SDAG-NEXT: buffer_store_dword v1, off, s[0:3], s4
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: s_mov_b32 s32, s34
-; GFX9-SDAG-NEXT: s_mov_b32 s34, s11
+; GFX9-SDAG-NEXT: s_mov_b32 s34, s10
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: buffer_load_dword v2, off, s[0:3], s33 offset:64 ; 4-byte Folded Reload
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-SDAG-NEXT: s_mov_b32 s33, s10
+; GFX9-SDAG-NEXT: s_mov_b32 s33, s9
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -2723,13 +2723,13 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) {
; GFX9-SDAG-LABEL: test_dynamic_stackalloc_device_control_flow:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-SDAG-NEXT: s_mov_b32 s10, s33
+; GFX9-SDAG-NEXT: s_mov_b32 s9, s33
; GFX9-SDAG-NEXT: s_add_i32 s33, s32, 0xfc0
; GFX9-SDAG-NEXT: s_and_b32 s33, s33, 0xfffff000
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: buffer_store_dword v2, off, s[0:3], s33 ; 4-byte Folded Spill
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-SDAG-NEXT: s_mov_b32 s11, s34
+; GFX9-SDAG-NEXT: s_mov_b32 s10, s34
; GFX9-SDAG-NEXT: v_cmp_ne_u32_e32 vcc, 0, v0
; GFX9-SDAG-NEXT: s_mov_b32 s34, s32
; GFX9-SDAG-NEXT: s_addk_i32 s32, 0x2000
@@ -2737,12 +2737,11 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) {
; GFX9-SDAG-NEXT: s_xor_b64 s[4:5], exec, s[4:5]
; GFX9-SDAG-NEXT: s_cbranch_execz .LBB15_2
; GFX9-SDAG-NEXT: ; %bb.1: ; %bb.1
-; GFX9-SDAG-NEXT: s_add_i32 s6, s32, 0xfff
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v0, v1, 2, 15
-; GFX9-SDAG-NEXT: s_and_b32 s8, s6, 0xfffff000
-; GFX9-SDAG-NEXT: v_and_b32_e32 v0, -16, v0
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, v1, 2, 15
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, 2
+; GFX9-SDAG-NEXT: v_and_b32_e32 v1, -16, v1
; GFX9-SDAG-NEXT: s_or_saveexec_b64 s[6:7], -1
-; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v0, s[6:7]
+; GFX9-SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v1, s[6:7]
; GFX9-SDAG-NEXT: s_nop 1
; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_shr:1 row_mask:0xf bank_mask:0xf
; GFX9-SDAG-NEXT: s_nop 1
@@ -2755,13 +2754,14 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) {
; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:15 row_mask:0xf bank_mask:0xf
; GFX9-SDAG-NEXT: s_nop 1
; GFX9-SDAG-NEXT: v_max_u32_dpp v2, v2, v2 row_bcast:31 row_mask:0xf bank_mask:0xf
-; GFX9-SDAG-NEXT: v_readlane_b32 s9, v2, 63
+; GFX9-SDAG-NEXT: v_readlane_b32 s8, v2, 63
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[6:7]
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v0, s8
-; GFX9-SDAG-NEXT: v_lshl_add_u32 v1, s9, 6, v0
-; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v1
-; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, 2
-; GFX9-SDAG-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen
+; GFX9-SDAG-NEXT: s_add_i32 s6, s32, 0xfff
+; GFX9-SDAG-NEXT: s_and_b32 s6, s6, 0xfffff000
+; GFX9-SDAG-NEXT: v_mov_b32_e32 v1, s6
+; GFX9-SDAG-NEXT: v_lshl_add_u32 v3, s8, 6, v1
+; GFX9-SDAG-NEXT: v_readfirstlane_b32 s32, v3
+; GFX9-SDAG-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: ; implicit-def: $vgpr31
; GFX9-SDAG-NEXT: .LBB15_2: ; %Flow
@@ -2797,11 +2797,11 @@ define void @test_dynamic_stackalloc_device_control_flow(i32 %n, i32 %m) {
; GFX9-SDAG-NEXT: .LBB15_4: ; %bb.2
; GFX9-SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-SDAG-NEXT: s_mov_b32 s32, s34
-; GFX9-SDAG-NEXT: s_mov_b32 s34, s11
+; GFX9-SDAG-NEXT: s_mov_b32 s34, s10
; GFX9-SDAG-NEXT: s_xor_saveexec_b64 s[4:5], -1
; GFX9-SDAG-NEXT: buffer_load_dword v2, off, s[0:3], s33 ; 4-byte Folded Reload
; GFX9-SDAG-NEXT: s_mov_b64 exec, s[4:5]
-; GFX9-SDAG-NEXT: s_mov_b32 s33, s10
+; GFX9-SDAG-NEXT: s_mov_b32 s33, s9
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
index 3ab0521a08061..d1efb7a35b122 100644
--- a/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
+++ b/llvm/test/CodeGen/AMDGPU/flat-scratch.ll
@@ -857,13 +857,13 @@ define void @store_load_vindex_foo(i32 %idx) {
; GFX9-LABEL: store_load_vindex_foo:
; GFX9: ; %bb.0: ; %bb
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_mov_b32 s0, s32
-; GFX9-NEXT: v_lshl_add_u32 v1, v0, 2, s0
-; GFX9-NEXT: v_mov_b32_e32 v2, 15
+; GFX9-NEXT: v_mov_b32_e32 v1, s32
+; GFX9-NEXT: v_lshl_add_u32 v2, v0, 2, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, 15
; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
-; GFX9-NEXT: scratch_store_dword v1, v2, off
+; GFX9-NEXT: scratch_store_dword v2, v3, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, s0
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v1
; GFX9-NEXT: scratch_load_dword v0, v0, off glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -915,13 +915,13 @@ define void @store_load_vindex_foo(i32 %idx) {
; GFX9-PAL-LABEL: store_load_vindex_foo:
; GFX9-PAL: ; %bb.0: ; %bb
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-PAL-NEXT: s_mov_b32 s0, s32
-; GFX9-PAL-NEXT: v_lshl_add_u32 v1, v0, 2, s0
-; GFX9-PAL-NEXT: v_mov_b32_e32 v2, 15
+; GFX9-PAL-NEXT: v_mov_b32_e32 v1, s32
+; GFX9-PAL-NEXT: v_lshl_add_u32 v2, v0, 2, v1
+; GFX9-PAL-NEXT: v_mov_b32_e32 v3, 15
; GFX9-PAL-NEXT: v_and_b32_e32 v0, 15, v0
-; GFX9-PAL-NEXT: scratch_store_dword v1, v2, off
+; GFX9-PAL-NEXT: scratch_store_dword v2, v3, off
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-PAL-NEXT: v_lshl_add_u32 v0, v0, 2, s0
+; GFX9-PAL-NEXT: v_lshl_add_u32 v0, v0, 2, v1
; GFX9-PAL-NEXT: scratch_load_dword v0, v0, off glc
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
; GFX9-PAL-NEXT: s_setpc_b64 s[30:31]
@@ -929,8 +929,8 @@ define void @store_load_vindex_foo(i32 %idx) {
; GFX942-LABEL: store_load_vindex_foo:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, s32
-; GFX942-NEXT: v_lshl_add_u32 v1, v0, 2, s0
+; GFX942-NEXT: v_mov_b32_e32 v1, s32
+; GFX942-NEXT: v_lshl_add_u32 v1, v0, 2, v1
; GFX942-NEXT: v_mov_b32_e32 v2, 15
; GFX942-NEXT: v_and_b32_e32 v0, 15, v0
; GFX942-NEXT: scratch_store_dword v1, v2, off sc0 sc1
@@ -2146,16 +2146,16 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
; GFX9-LABEL: store_load_vindex_small_offset_foo:
; GFX9: ; %bb.0: ; %bb
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_add_i32 s1, s32, 0x100
; GFX9-NEXT: scratch_load_dword v1, off, s32 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_mov_b32 s0, s1
-; GFX9-NEXT: v_lshl_add_u32 v1, v0, 2, s0
-; GFX9-NEXT: v_mov_b32_e32 v2, 15
+; GFX9-NEXT: s_add_i32 s0, s32, 0x100
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: v_lshl_add_u32 v2, v0, 2, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, 15
; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
-; GFX9-NEXT: scratch_store_dword v1, v2, off
+; GFX9-NEXT: scratch_store_dword v2, v3, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, s0
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v1
; GFX9-NEXT: scratch_load_dword v0, v0, off glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -2214,16 +2214,16 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
; GFX9-PAL-LABEL: store_load_vindex_small_offset_foo:
; GFX9-PAL: ; %bb.0: ; %bb
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-PAL-NEXT: s_add_i32 s1, s32, 0x100
; GFX9-PAL-NEXT: scratch_load_dword v1, off, s32 glc
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-PAL-NEXT: s_mov_b32 s0, s1
-; GFX9-PAL-NEXT: v_lshl_add_u32 v1, v0, 2, s0
-; GFX9-PAL-NEXT: v_mov_b32_e32 v2, 15
+; GFX9-PAL-NEXT: s_add_i32 s0, s32, 0x100
+; GFX9-PAL-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-PAL-NEXT: v_lshl_add_u32 v2, v0, 2, v1
+; GFX9-PAL-NEXT: v_mov_b32_e32 v3, 15
; GFX9-PAL-NEXT: v_and_b32_e32 v0, 15, v0
-; GFX9-PAL-NEXT: scratch_store_dword v1, v2, off
+; GFX9-PAL-NEXT: scratch_store_dword v2, v3, off
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-PAL-NEXT: v_lshl_add_u32 v0, v0, 2, s0
+; GFX9-PAL-NEXT: v_lshl_add_u32 v0, v0, 2, v1
; GFX9-PAL-NEXT: scratch_load_dword v0, v0, off glc
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
; GFX9-PAL-NEXT: s_setpc_b64 s[30:31]
@@ -2231,11 +2231,11 @@ define void @store_load_vindex_small_offset_foo(i32 %idx) {
; GFX942-LABEL: store_load_vindex_small_offset_foo:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_add_i32 s1, s32, 0x100
; GFX942-NEXT: scratch_load_dword v1, off, s32 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, s1
-; GFX942-NEXT: v_lshl_add_u32 v1, v0, 2, s0
+; GFX942-NEXT: s_add_i32 s0, s32, 0x100
+; GFX942-NEXT: v_mov_b32_e32 v1, s0
+; GFX942-NEXT: v_lshl_add_u32 v1, v0, 2, v1
; GFX942-NEXT: v_mov_b32_e32 v2, 15
; GFX942-NEXT: v_and_b32_e32 v0, 15, v0
; GFX942-NEXT: scratch_store_dword v1, v2, off sc0 sc1
@@ -3447,16 +3447,16 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
; GFX9-LABEL: store_load_vindex_large_offset_foo:
; GFX9: ; %bb.0: ; %bb
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_add_i32 s1, s32, 0x4004
; GFX9-NEXT: scratch_load_dword v1, off, s32 offset:4 glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: s_mov_b32 s0, s1
-; GFX9-NEXT: v_lshl_add_u32 v1, v0, 2, s0
-; GFX9-NEXT: v_mov_b32_e32 v2, 15
+; GFX9-NEXT: s_add_i32 s0, s32, 0x4004
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: v_lshl_add_u32 v2, v0, 2, v1
+; GFX9-NEXT: v_mov_b32_e32 v3, 15
; GFX9-NEXT: v_and_b32_e32 v0, 15, v0
-; GFX9-NEXT: scratch_store_dword v1, v2, off
+; GFX9-NEXT: scratch_store_dword v2, v3, off
; GFX9-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, s0
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v1
; GFX9-NEXT: scratch_load_dword v0, v0, off glc
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -3516,16 +3516,16 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
; GFX9-PAL-LABEL: store_load_vindex_large_offset_foo:
; GFX9-PAL: ; %bb.0: ; %bb
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-PAL-NEXT: s_add_i32 s1, s32, 0x4004
; GFX9-PAL-NEXT: scratch_load_dword v1, off, s32 offset:4 glc
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-PAL-NEXT: s_mov_b32 s0, s1
-; GFX9-PAL-NEXT: v_lshl_add_u32 v1, v0, 2, s0
-; GFX9-PAL-NEXT: v_mov_b32_e32 v2, 15
+; GFX9-PAL-NEXT: s_add_i32 s0, s32, 0x4004
+; GFX9-PAL-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-PAL-NEXT: v_lshl_add_u32 v2, v0, 2, v1
+; GFX9-PAL-NEXT: v_mov_b32_e32 v3, 15
; GFX9-PAL-NEXT: v_and_b32_e32 v0, 15, v0
-; GFX9-PAL-NEXT: scratch_store_dword v1, v2, off
+; GFX9-PAL-NEXT: scratch_store_dword v2, v3, off
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
-; GFX9-PAL-NEXT: v_lshl_add_u32 v0, v0, 2, s0
+; GFX9-PAL-NEXT: v_lshl_add_u32 v0, v0, 2, v1
; GFX9-PAL-NEXT: scratch_load_dword v0, v0, off glc
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
; GFX9-PAL-NEXT: s_setpc_b64 s[30:31]
@@ -3533,11 +3533,11 @@ define void @store_load_vindex_large_offset_foo(i32 %idx) {
; GFX942-LABEL: store_load_vindex_large_offset_foo:
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX942-NEXT: s_add_i32 s1, s32, 0x4004
; GFX942-NEXT: scratch_load_dword v1, off, s32 offset:4 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
-; GFX942-NEXT: s_mov_b32 s0, s1
-; GFX942-NEXT: v_lshl_add_u32 v1, v0, 2, s0
+; GFX942-NEXT: s_add_i32 s0, s32, 0x4004
+; GFX942-NEXT: v_mov_b32_e32 v1, s0
+; GFX942-NEXT: v_lshl_add_u32 v1, v0, 2, v1
; GFX942-NEXT: v_mov_b32_e32 v2, 15
; GFX942-NEXT: v_and_b32_e32 v0, 15, v0
; GFX942-NEXT: scratch_store_dword v1, v2, off sc0 sc1
@@ -3940,12 +3940,12 @@ define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
; GFX9: ; %bb.0: ; %bb
; GFX9-NEXT: s_load_dword s0, s[4:5], 0x24
; GFX9-NEXT: s_add_u32 flat_scratch_lo, s8, s13
-; GFX9-NEXT: s_mov_b32 s1, 0
+; GFX9-NEXT: v_mov_b32_e32 v1, 0
; GFX9-NEXT: s_addc_u32 flat_scratch_hi, s9, 0
-; GFX9-NEXT: v_mov_b32_e32 v1, 15
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_add_u32_e32 v0, s0, v0
-; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v1
+; GFX9-NEXT: v_mov_b32_e32 v1, 15
; GFX9-NEXT: scratch_store_dword v0, v1, off offset:1024
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: scratch_load_dword v0, v0, off offset:1024 glc
@@ -4001,15 +4001,15 @@ define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
; GFX9-PAL-NEXT: s_getpc_b64 s[12:13]
; GFX9-PAL-NEXT: s_mov_b32 s12, s0
; GFX9-PAL-NEXT: s_load_dwordx2 s[12:13], s[12:13], 0x0
-; GFX9-PAL-NEXT: s_mov_b32 s1, 0
+; GFX9-PAL-NEXT: v_mov_b32_e32 v1, 0
; GFX9-PAL-NEXT: s_load_dword s0, s[4:5], 0x0
-; GFX9-PAL-NEXT: v_mov_b32_e32 v1, 15
; GFX9-PAL-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-PAL-NEXT: s_and_b32 s13, s13, 0xffff
; GFX9-PAL-NEXT: s_add_u32 flat_scratch_lo, s12, s11
; GFX9-PAL-NEXT: v_add_u32_e32 v0, s0, v0
; GFX9-PAL-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
-; GFX9-PAL-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX9-PAL-NEXT: v_lshl_add_u32 v0, v0, 2, v1
+; GFX9-PAL-NEXT: v_mov_b32_e32 v1, 15
; GFX9-PAL-NEXT: scratch_store_dword v0, v1, off offset:1024
; GFX9-PAL-NEXT: s_waitcnt vmcnt(0)
; GFX9-PAL-NEXT: scratch_load_dword v0, v0, off offset:1024 glc
@@ -4020,11 +4020,11 @@ define amdgpu_kernel void @store_load_vidx_sidx_offset(i32 %sidx) {
; GFX942: ; %bb.0: ; %bb
; GFX942-NEXT: s_load_dword s0, s[4:5], 0x24
; GFX942-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX942-NEXT: s_mov_b32 s1, 0
-; GFX942-NEXT: v_mov_b32_e32 v1, 15
+; GFX942-NEXT: v_mov_b32_e32 v1, 0
; GFX942-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NEXT: v_add_u32_e32 v0, s0, v0
-; GFX942-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX942-NEXT: v_lshl_add_u32 v0, v0, 2, v1
+; GFX942-NEXT: v_mov_b32_e32 v1, 15
; GFX942-NEXT: scratch_store_dword v0, v1, off offset:1024 sc0 sc1
; GFX942-NEXT: s_waitcnt vmcnt(0)
; GFX942-NEXT: scratch_load_dword v0, v0, off offset:1024 sc0 sc1
diff --git a/llvm/test/CodeGen/AMDGPU/gep-flags-stack-offsets.ll b/llvm/test/CodeGen/AMDGPU/gep-flags-stack-offsets.ll
index 61902b5fd4661..b5f0b2ff9ef4c 100644
--- a/llvm/test/CodeGen/AMDGPU/gep-flags-stack-offsets.ll
+++ b/llvm/test/CodeGen/AMDGPU/gep-flags-stack-offsets.ll
@@ -18,8 +18,8 @@ define void @gep_noflags_alloca(i32 %idx, i32 %val) #0 {
; GFX9-LABEL: gep_noflags_alloca:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s32, 6
-; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, s4
+; GFX9-NEXT: v_lshrrev_b32_e64 v2, 6, s32
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -45,8 +45,8 @@ define void @gep_inbounds_alloca(i32 %idx, i32 %val) #0 {
; GFX9-LABEL: gep_inbounds_alloca:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s32, 6
-; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, s4
+; GFX9-NEXT: v_lshrrev_b32_e64 v2, 6, s32
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -72,8 +72,8 @@ define void @gep_nuw_alloca(i32 %idx, i32 %val) #0 {
; GFX9-LABEL: gep_nuw_alloca:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s32, 6
-; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, s4
+; GFX9-NEXT: v_lshrrev_b32_e64 v2, 6, s32
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -99,8 +99,8 @@ define void @gep_nusw_alloca(i32 %idx, i32 %val) #0 {
; GFX9-LABEL: gep_nusw_alloca:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s32, 6
-; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, s4
+; GFX9-NEXT: v_lshrrev_b32_e64 v2, 6, s32
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -126,8 +126,8 @@ define void @gep_inbounds_nuw_alloca(i32 %idx, i32 %val) #0 {
; GFX9-LABEL: gep_inbounds_nuw_alloca:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s32, 6
-; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, s4
+; GFX9-NEXT: v_lshrrev_b32_e64 v2, 6, s32
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
@@ -153,8 +153,8 @@ define void @gep_nusw_nuw_alloca(i32 %idx, i32 %val) #0 {
; GFX9-LABEL: gep_nusw_nuw_alloca:
; GFX9: ; %bb.0:
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX9-NEXT: s_lshr_b32 s4, s32, 6
-; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, s4
+; GFX9-NEXT: v_lshrrev_b32_e64 v2, 6, s32
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 2, v2
; GFX9-NEXT: buffer_store_dword v1, v0, s[0:3], 0 offen offset:16
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/hip.extern.shared.array.ll b/llvm/test/CodeGen/AMDGPU/hip.extern.shared.array.ll
index d24b3a23cb9cd..8bd6c0f2652cf 100644
--- a/llvm/test/CodeGen/AMDGPU/hip.extern.shared.array.ll
+++ b/llvm/test/CodeGen/AMDGPU/hip.extern.shared.array.ll
@@ -22,7 +22,7 @@ define amdgpu_kernel void @dynamic_shared_array_0(ptr addrspace(1) %out) {
}
; CHECK-LABEL: {{^}}dynamic_shared_array_1:
-; CHECK: s_movk_i32 [[DYNLDS:s[0-9]+]], 0xc00
+; CHECK: v_mov_b32_e32 [[DYNLDS:v[0-9]+]], 0xc00
; CHECK: v_lshl_add_u32 {{v[0-9]+}}, {{v[0-9]+}}, 2, [[DYNLDS]]
define amdgpu_kernel void @dynamic_shared_array_1(ptr addrspace(1) %out, i32 %cond) {
entry:
@@ -49,7 +49,7 @@ endif: ; preds = %else, %if
}
; CHECK-LABEL: {{^}}dynamic_shared_array_2:
-; CHECK: s_movk_i32 [[DYNLDS:s[0-9]+]], 0x4000
+; CHECK: v_mov_b32_e32 [[DYNLDS:v[0-9]+]], 0x4000
; CHECK: v_lshl_add_u32 {{v[0-9]+}}, {{v[0-9]+}}, 2, [[DYNLDS]]
define amdgpu_kernel void @dynamic_shared_array_2(i32 %idx) {
%tid.x = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -64,7 +64,7 @@ define amdgpu_kernel void @dynamic_shared_array_2(i32 %idx) {
; The offset to the dynamic shared memory array should be aligned on the type
; specified.
; CHECK-LABEL: {{^}}dynamic_shared_array_3:
-; CHECK: s_movk_i32 [[DYNLDS:s[0-9]+]], 0x44
+; CHECK: v_mov_b32_e32 [[DYNLDS:v[0-9]+]], 0x44
; CHECK: v_lshl_add_u32 {{v[0-9]+}}, {{v[0-9]+}}, 2, [[DYNLDS]]
define amdgpu_kernel void @dynamic_shared_array_3(i32 %idx) {
%tid.x = tail call i32 @llvm.amdgcn.workitem.id.x()
@@ -80,7 +80,7 @@ define amdgpu_kernel void @dynamic_shared_array_3(i32 %idx) {
; The offset to the dynamic shared memory array should be aligned on the
; maximal one.
; CHECK-LABEL: {{^}}dynamic_shared_array_4:
-; CHECK: s_movk_i32 [[DYNLDS:s[0-9]+]], 0x48
+; CHECK: v_mov_b32_e32 [[DYNLDS:v[0-9]+]], 0x48
; CHECK-DAG: v_lshl_add_u32 {{v[0-9]+}}, {{v[0-9]+}}, 2, [[DYNLDS]]
; CHECK-DAG: v_lshl_add_u32 {{v[0-9]+}}, {{v[0-9]+}}, 3, [[DYNLDS]]
define amdgpu_kernel void @dynamic_shared_array_4(i32 %idx) {
@@ -99,7 +99,7 @@ define amdgpu_kernel void @dynamic_shared_array_4(i32 %idx) {
; Honor the explicit alignment from the specified variable.
; CHECK-LABEL: {{^}}dynamic_shared_array_5:
-; CHECK: s_movk_i32 [[DYNLDS:s[0-9]+]], 0x44
+; CHECK: v_mov_b32_e32 [[DYNLDS:v[0-9]+]], 0x44
; CHECK-DAG: v_lshl_add_u32 {{v[0-9]+}}, {{v[0-9]+}}, 2, [[DYNLDS]]
; CHECK-DAG: v_lshl_add_u32 {{v[0-9]+}}, {{v[0-9]+}}, 3, [[DYNLDS]]
define amdgpu_kernel void @dynamic_shared_array_5(i32 %idx) {
@@ -118,7 +118,7 @@ define amdgpu_kernel void @dynamic_shared_array_5(i32 %idx) {
; Honor the explicit alignment from the specified variable.
; CHECK-LABEL: {{^}}dynamic_shared_array_6:
-; CHECK: s_movk_i32 [[DYNLDS:s[0-9]+]], 0x50
+; CHECK: v_mov_b32_e32 [[DYNLDS:v[0-9]+]], 0x50
; CHECK-DAG: v_lshl_add_u32 {{v[0-9]+}}, {{v[0-9]+}}, 2, [[DYNLDS]]
; CHECK-DAG: v_lshl_add_u32 {{v[0-9]+}}, {{v[0-9]+}}, 3, [[DYNLDS]]
define amdgpu_kernel void @dynamic_shared_array_6(i32 %idx) {
diff --git a/llvm/test/CodeGen/AMDGPU/lds-relocs.ll b/llvm/test/CodeGen/AMDGPU/lds-relocs.ll
index 447cb62643384..91489d76b18f6 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-relocs.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-relocs.ll
@@ -32,7 +32,7 @@
; ELF-NEXT: }
; GCN-LABEL: {{^}}test_basic:
-; GCN: s_mov_b32 s0, lds.external at abs32@lo ; encoding: [0xff,0x00,0x80,0xbe,A,A,A,A]
+; GCN: v_mov_b32_e32 v1, lds.external at abs32@lo ; encoding: [0xff,0x02,0x02,0x7e,A,A,A,A]
; GCN-NEXT: ; fixup A - offset: 4, value: lds.external at abs32@lo, kind: FK_Data_4{{$}}
;
; GCN: s_lshl2_add_u32 s0, s2, lds.defined at abs32@lo ; encoding: [0x02,0xff,0x80,0x97,A,A,A,A]
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
index 58890b0b87ab0..127e5b247f0cc 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-nontemporal.ll
@@ -309,11 +309,12 @@ define amdgpu_kernel void @local_nontemporal_load_1(
; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
-; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x0
+; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v1, 2, s6
+; GFX10-WGP-NEXT: s_mov_b32 s6, 2
+; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v1, s6, s7
; GFX10-WGP-NEXT: ds_read_b32 v1, v1
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
@@ -325,11 +326,12 @@ define amdgpu_kernel void @local_nontemporal_load_1(
; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
-; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x0
+; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-CU-NEXT: v_lshl_add_u32 v1, v1, 2, s6
+; GFX10-CU-NEXT: s_mov_b32 s6, 2
+; GFX10-CU-NEXT: v_lshl_add_u32 v1, v1, s6, s7
; GFX10-CU-NEXT: ds_read_b32 v1, v1
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
@@ -367,13 +369,15 @@ define amdgpu_kernel void @local_nontemporal_load_1(
; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x0
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s7, 0x3ff
-; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v1, v1, s7
-; GFX90A-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v1, 2, s6
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s6, 0x3ff
+; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v1, v1, s6
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s6, 2
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
+; GFX90A-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v1, s6, v2
; GFX90A-NOTTGSPLIT-NEXT: ds_read_b32 v1, v1
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
@@ -385,13 +389,15 @@ define amdgpu_kernel void @local_nontemporal_load_1(
; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
-; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x0
+; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x0
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-TGSPLIT-NEXT: s_mov_b32 s7, 0x3ff
-; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v1, v1, s7
-; GFX90A-TGSPLIT-NEXT: v_lshl_add_u32 v1, v1, 2, s6
+; GFX90A-TGSPLIT-NEXT: s_mov_b32 s6, 0x3ff
+; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v1, v1, s6
+; GFX90A-TGSPLIT-NEXT: s_mov_b32 s6, 2
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
+; GFX90A-TGSPLIT-NEXT: v_lshl_add_u32 v1, v1, s6, v2
; GFX90A-TGSPLIT-NEXT: ds_read_b32 v1, v1
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
@@ -403,13 +409,15 @@ define amdgpu_kernel void @local_nontemporal_load_1(
; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX942-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s3, 0x3ff
-; GFX942-NOTTGSPLIT-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX942-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s2, 0x3ff
+; GFX942-NOTTGSPLIT-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s2, 2
+; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v1, s2, v2
; GFX942-NOTTGSPLIT-NEXT: ds_read_b32 v1, v1
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1]
@@ -421,13 +429,15 @@ define amdgpu_kernel void @local_nontemporal_load_1(
; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX942-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-TGSPLIT-NEXT: s_mov_b32 s3, 0x3ff
-; GFX942-TGSPLIT-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX942-TGSPLIT-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX942-TGSPLIT-NEXT: s_mov_b32 s2, 0x3ff
+; GFX942-TGSPLIT-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX942-TGSPLIT-NEXT: s_mov_b32 s2, 2
+; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-TGSPLIT-NEXT: v_lshl_add_u32 v1, v1, s2, v2
; GFX942-TGSPLIT-NEXT: ds_read_b32 v1, v1
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1]
@@ -439,13 +449,14 @@ define amdgpu_kernel void @local_nontemporal_load_1(
; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
-; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x0
+; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-WGP-NEXT: s_mov_b32 s3, 0x3ff
-; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff
+; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX11-WGP-NEXT: s_mov_b32 s2, 2
+; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX11-WGP-NEXT: ds_load_b32 v1, v1
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -457,13 +468,14 @@ define amdgpu_kernel void @local_nontemporal_load_1(
; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
-; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x0
+; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-CU-NEXT: s_mov_b32 s3, 0x3ff
-; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX11-CU-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff
+; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX11-CU-NEXT: s_mov_b32 s2, 2
+; GFX11-CU-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX11-CU-NEXT: ds_load_b32 v1, v1
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -475,13 +487,15 @@ define amdgpu_kernel void @local_nontemporal_load_1(
; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
-; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x0
+; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-WGP-NEXT: s_mov_b32 s3, 0x3ff
-; GFX12-WGP-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX12-WGP-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX12-WGP-NEXT: s_mov_b32 s2, 0x3ff
+; GFX12-WGP-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX12-WGP-NEXT: s_mov_b32 s2, 2
+; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-WGP-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX12-WGP-NEXT: ds_load_b32 v1, v1
; GFX12-WGP-NEXT: s_wait_dscnt 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
@@ -494,13 +508,15 @@ define amdgpu_kernel void @local_nontemporal_load_1(
; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
-; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x0
+; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-CU-NEXT: s_mov_b32 s3, 0x3ff
-; GFX12-CU-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX12-CU-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX12-CU-NEXT: s_mov_b32 s2, 0x3ff
+; GFX12-CU-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX12-CU-NEXT: s_mov_b32 s2, 2
+; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-CU-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX12-CU-NEXT: ds_load_b32 v1, v1
; GFX12-CU-NEXT: s_wait_dscnt 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
@@ -514,13 +530,14 @@ define amdgpu_kernel void @local_nontemporal_load_1(
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_mov_b32 s3, 0x3ff
-; GFX1250-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX1250-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX1250-NEXT: s_mov_b32 s2, 0x3ff
+; GFX1250-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX1250-NEXT: s_mov_b32 s2, 2
+; GFX1250-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX1250-NEXT: ds_load_b32 v1, v1
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -796,11 +813,13 @@ define amdgpu_kernel void @local_nontemporal_store_1(
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
-; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x8
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX10-WGP-NEXT: v_lshl_add_u32 v0, v0, 2, s5
+; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x8
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX10-WGP-NEXT: s_mov_b32 s5, 2
+; GFX10-WGP-NEXT: v_lshl_add_u32 v0, v0, s5, s6
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s4
; GFX10-WGP-NEXT: ds_write_b32 v0, v1
@@ -811,11 +830,13 @@ define amdgpu_kernel void @local_nontemporal_store_1(
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
-; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x8
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX10-CU-NEXT: v_lshl_add_u32 v0, v0, 2, s5
+; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x8
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX10-CU-NEXT: s_mov_b32 s5, 2
+; GFX10-CU-NEXT: v_lshl_add_u32 v0, v0, s5, s6
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: v_mov_b32_e32 v1, s4
; GFX10-CU-NEXT: ds_write_b32 v0, v1
@@ -844,13 +865,16 @@ define amdgpu_kernel void @local_nontemporal_store_1(
; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x8
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s6, 0x3ff
-; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s6
-; GFX90A-NOTTGSPLIT-NEXT: v_lshl_add_u32 v0, v0, 2, s5
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s5, 0x3ff
+; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s5
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s5, 2
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-NOTTGSPLIT-NEXT: v_lshl_add_u32 v0, v0, s5, v1
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s4
; GFX90A-NOTTGSPLIT-NEXT: ds_write_b32 v0, v1
@@ -861,13 +885,16 @@ define amdgpu_kernel void @local_nontemporal_store_1(
; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
-; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x8
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX90A-TGSPLIT-NEXT: s_mov_b32 s6, 0x3ff
-; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s6
-; GFX90A-TGSPLIT-NEXT: v_lshl_add_u32 v0, v0, 2, s5
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
+; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX90A-TGSPLIT-NEXT: s_mov_b32 s5, 0x3ff
+; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s5
+; GFX90A-TGSPLIT-NEXT: s_mov_b32 s5, 2
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-TGSPLIT-NEXT: v_lshl_add_u32 v0, v0, s5, v1
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s4
; GFX90A-TGSPLIT-NEXT: ds_write_b32 v0, v1
@@ -878,13 +905,16 @@ define amdgpu_kernel void @local_nontemporal_store_1(
; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
-; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x8
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s2, 0x3ff
-; GFX942-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX942-NOTTGSPLIT-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
+; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x0
+; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s1, 0x3ff
+; GFX942-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s1, 2
+; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NOTTGSPLIT-NEXT: v_lshl_add_u32 v0, v0, s1, v1
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s0
; GFX942-NOTTGSPLIT-NEXT: ds_write_b32 v0, v1
@@ -895,13 +925,16 @@ define amdgpu_kernel void @local_nontemporal_store_1(
; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
-; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x8
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX942-TGSPLIT-NEXT: s_mov_b32 s2, 0x3ff
-; GFX942-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX942-TGSPLIT-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
+; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x0
+; GFX942-TGSPLIT-NEXT: s_mov_b32 s1, 0x3ff
+; GFX942-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX942-TGSPLIT-NEXT: s_mov_b32 s1, 2
+; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-TGSPLIT-NEXT: v_lshl_add_u32 v0, v0, s1, v1
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s0
; GFX942-TGSPLIT-NEXT: ds_write_b32 v0, v1
@@ -912,13 +945,15 @@ define amdgpu_kernel void @local_nontemporal_store_1(
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
-; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x8
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff
-; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX11-WGP-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX11-WGP-NEXT: s_mov_b32 s1, 0x3ff
+; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX11-WGP-NEXT: s_mov_b32 s1, 2
+; GFX11-WGP-NEXT: v_lshl_add_u32 v0, v0, s1, s2
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s0
; GFX11-WGP-NEXT: ds_store_b32 v0, v1
@@ -929,13 +964,15 @@ define amdgpu_kernel void @local_nontemporal_store_1(
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
-; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x8
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff
-; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX11-CU-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX11-CU-NEXT: s_mov_b32 s1, 0x3ff
+; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX11-CU-NEXT: s_mov_b32 s1, 2
+; GFX11-CU-NEXT: v_lshl_add_u32 v0, v0, s1, s2
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: v_mov_b32_e32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
@@ -946,13 +983,16 @@ define amdgpu_kernel void @local_nontemporal_store_1(
; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
-; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x8
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX12-WGP-NEXT: s_mov_b32 s2, 0x3ff
-; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX12-WGP-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_mov_b32 s1, 0x3ff
+; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX12-WGP-NEXT: s_mov_b32 s1, 2
+; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-WGP-NEXT: v_lshl_add_u32 v0, v0, s1, s2
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s0
; GFX12-WGP-NEXT: ds_store_b32 v0, v1
@@ -963,13 +1003,16 @@ define amdgpu_kernel void @local_nontemporal_store_1(
; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
-; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x8
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX12-CU-NEXT: s_mov_b32 s2, 0x3ff
-; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX12-CU-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_mov_b32 s1, 0x3ff
+; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX12-CU-NEXT: s_mov_b32 s1, 2
+; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-CU-NEXT: v_lshl_add_u32 v0, v0, s1, s2
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: v_mov_b32_e32 v1, s0
; GFX12-CU-NEXT: ds_store_b32 v0, v1
@@ -981,14 +1024,16 @@ define amdgpu_kernel void @local_nontemporal_store_1(
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
-; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
-; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_load_b32 s0, s[2:3], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x8 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_mov_b32 s2, 0x3ff
-; GFX1250-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX1250-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX1250-NEXT: s_mov_b32 s1, 0x3ff
+; GFX1250-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX1250-NEXT: s_mov_b32 s1, 2
+; GFX1250-NEXT: v_lshl_add_u32 v0, v0, s1, s2
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v1, s0
; GFX1250-NEXT: ds_store_b32 v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
index c0420028488dc..2a16255155e18 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-local-volatile.ll
@@ -245,11 +245,12 @@ define amdgpu_kernel void @local_volatile_load_1(
; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
-; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x0
+; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v1, 2, s6
+; GFX10-WGP-NEXT: s_mov_b32 s6, 2
+; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v1, s6, s7
; GFX10-WGP-NEXT: ds_read_b32 v1, v1
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
@@ -261,11 +262,12 @@ define amdgpu_kernel void @local_volatile_load_1(
; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
-; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x0
+; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-CU-NEXT: v_lshl_add_u32 v1, v1, 2, s6
+; GFX10-CU-NEXT: s_mov_b32 s6, 2
+; GFX10-CU-NEXT: v_lshl_add_u32 v1, v1, s6, s7
; GFX10-CU-NEXT: ds_read_b32 v1, v1
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
@@ -303,13 +305,14 @@ define amdgpu_kernel void @local_volatile_load_1(
; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
-; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x0
+; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-WGP-NEXT: s_mov_b32 s3, 0x3ff
-; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff
+; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX11-WGP-NEXT: s_mov_b32 s2, 2
+; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX11-WGP-NEXT: ds_load_b32 v1, v1
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -321,13 +324,14 @@ define amdgpu_kernel void @local_volatile_load_1(
; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
-; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x0
+; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-CU-NEXT: s_mov_b32 s3, 0x3ff
-; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX11-CU-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff
+; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX11-CU-NEXT: s_mov_b32 s2, 2
+; GFX11-CU-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX11-CU-NEXT: ds_load_b32 v1, v1
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -339,13 +343,15 @@ define amdgpu_kernel void @local_volatile_load_1(
; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
-; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x0
+; GFX12-WGP-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX12-WGP-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-WGP-NEXT: s_mov_b32 s3, 0x3ff
-; GFX12-WGP-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX12-WGP-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX12-WGP-NEXT: s_mov_b32 s2, 0x3ff
+; GFX12-WGP-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX12-WGP-NEXT: s_mov_b32 s2, 2
+; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-WGP-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX12-WGP-NEXT: ds_load_b32 v1, v1
; GFX12-WGP-NEXT: s_wait_dscnt 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
@@ -358,13 +364,15 @@ define amdgpu_kernel void @local_volatile_load_1(
; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
-; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x0
+; GFX12-CU-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX12-CU-NEXT: v_mov_b32_e32 v0, 0
-; GFX12-CU-NEXT: s_mov_b32 s3, 0x3ff
-; GFX12-CU-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX12-CU-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX12-CU-NEXT: s_mov_b32 s2, 0x3ff
+; GFX12-CU-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX12-CU-NEXT: s_mov_b32 s2, 2
+; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-CU-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX12-CU-NEXT: ds_load_b32 v1, v1
; GFX12-CU-NEXT: s_wait_dscnt 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
@@ -378,13 +386,14 @@ define amdgpu_kernel void @local_volatile_load_1(
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
-; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b32 s3, s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_mov_b32_e32 v0, 0
-; GFX1250-NEXT: s_mov_b32 s3, 0x3ff
-; GFX1250-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX1250-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX1250-NEXT: s_mov_b32 s2, 0x3ff
+; GFX1250-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX1250-NEXT: s_mov_b32 s2, 2
+; GFX1250-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX1250-NEXT: ds_load_b32 v1, v1
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -600,11 +609,13 @@ define amdgpu_kernel void @local_volatile_store_1(
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
-; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x8
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX10-WGP-NEXT: v_lshl_add_u32 v0, v0, 2, s5
+; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x8
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX10-WGP-NEXT: s_mov_b32 s5, 2
+; GFX10-WGP-NEXT: v_lshl_add_u32 v0, v0, s5, s6
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: v_mov_b32_e32 v1, s4
; GFX10-WGP-NEXT: ds_write_b32 v0, v1
@@ -615,11 +626,13 @@ define amdgpu_kernel void @local_volatile_store_1(
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
-; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x8
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX10-CU-NEXT: v_lshl_add_u32 v0, v0, 2, s5
+; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x8
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX10-CU-NEXT: s_mov_b32 s5, 2
+; GFX10-CU-NEXT: v_lshl_add_u32 v0, v0, s5, s6
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: v_mov_b32_e32 v1, s4
; GFX10-CU-NEXT: ds_write_b32 v0, v1
@@ -648,13 +661,15 @@ define amdgpu_kernel void @local_volatile_store_1(
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
-; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x8
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff
-; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX11-WGP-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX11-WGP-NEXT: s_mov_b32 s1, 0x3ff
+; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX11-WGP-NEXT: s_mov_b32 s1, 2
+; GFX11-WGP-NEXT: v_lshl_add_u32 v0, v0, s1, s2
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: v_mov_b32_e32 v1, s0
; GFX11-WGP-NEXT: ds_store_b32 v0, v1
@@ -665,13 +680,15 @@ define amdgpu_kernel void @local_volatile_store_1(
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
-; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x8
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff
-; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX11-CU-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX11-CU-NEXT: s_mov_b32 s1, 0x3ff
+; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX11-CU-NEXT: s_mov_b32 s1, 2
+; GFX11-CU-NEXT: v_lshl_add_u32 v0, v0, s1, s2
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: v_mov_b32_e32 v1, s0
; GFX11-CU-NEXT: ds_store_b32 v0, v1
@@ -682,13 +699,16 @@ define amdgpu_kernel void @local_volatile_store_1(
; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX12-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
-; GFX12-WGP-NEXT: s_load_b32 s1, s[4:5], 0x8
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
-; GFX12-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX12-WGP-NEXT: s_mov_b32 s2, 0x3ff
-; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX12-WGP-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX12-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX12-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
+; GFX12-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-WGP-NEXT: s_mov_b32 s1, 0x3ff
+; GFX12-WGP-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX12-WGP-NEXT: s_mov_b32 s1, 2
+; GFX12-WGP-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-WGP-NEXT: v_lshl_add_u32 v0, v0, s1, s2
; GFX12-WGP-NEXT: s_wait_kmcnt 0x0
; GFX12-WGP-NEXT: v_mov_b32_e32 v1, s0
; GFX12-WGP-NEXT: ds_store_b32 v0, v1
@@ -699,13 +719,16 @@ define amdgpu_kernel void @local_volatile_store_1(
; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX12-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
-; GFX12-CU-NEXT: s_load_b32 s1, s[4:5], 0x8
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
-; GFX12-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX12-CU-NEXT: s_mov_b32 s2, 0x3ff
-; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX12-CU-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX12-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX12-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX12-CU-NEXT: s_wait_kmcnt 0x0
+; GFX12-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX12-CU-NEXT: s_mov_b32 s1, 0x3ff
+; GFX12-CU-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX12-CU-NEXT: s_mov_b32 s1, 2
+; GFX12-CU-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-CU-NEXT: v_lshl_add_u32 v0, v0, s1, s2
; GFX12-CU-NEXT: s_wait_kmcnt 0x0
; GFX12-CU-NEXT: v_mov_b32_e32 v1, s0
; GFX12-CU-NEXT: ds_store_b32 v0, v1
@@ -717,14 +740,16 @@ define amdgpu_kernel void @local_volatile_store_1(
; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b32 s0, s[4:5], 0x8 nv
-; GFX1250-NEXT: s_load_b64 s[2:3], s[4:5], 0x0 nv
-; GFX1250-NEXT: s_load_b32 s1, s[4:5], 0x8 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_load_b32 s0, s[2:3], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
+; GFX1250-NEXT: s_load_b32 s2, s[4:5], 0x8 nv
+; GFX1250-NEXT: s_wait_kmcnt 0x0
+; GFX1250-NEXT: s_load_b32 s0, s[0:1], 0x0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_mov_b32 s2, 0x3ff
-; GFX1250-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX1250-NEXT: v_lshl_add_u32 v0, v0, 2, s1
+; GFX1250-NEXT: s_mov_b32 s1, 0x3ff
+; GFX1250-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX1250-NEXT: s_mov_b32 s1, 2
+; GFX1250-NEXT: v_lshl_add_u32 v0, v0, s1, s2
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_mov_b32_e32 v1, s0
; GFX1250-NEXT: ds_store_b32 v0, v1
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
index 69aa48ff1daaf..2e0aa7f2fbd0d 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-nontemporal.ll
@@ -327,11 +327,12 @@ define amdgpu_kernel void @private_nontemporal_load_1(
; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
-; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x0
+; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v1, 2, s6
+; GFX10-WGP-NEXT: s_mov_b32 s6, 2
+; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v1, s6, s7
; GFX10-WGP-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen slc
; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
@@ -345,11 +346,12 @@ define amdgpu_kernel void @private_nontemporal_load_1(
; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
-; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x0
+; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-CU-NEXT: v_lshl_add_u32 v1, v1, 2, s6
+; GFX10-CU-NEXT: s_mov_b32 s6, 2
+; GFX10-CU-NEXT: v_lshl_add_u32 v1, v1, s6, s7
; GFX10-CU-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen slc
; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
@@ -394,13 +396,15 @@ define amdgpu_kernel void @private_nontemporal_load_1(
; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x0
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s7, 0x3ff
-; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v1, v1, s7
-; GFX90A-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v1, 2, s6
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s6, 0x3ff
+; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v1, v1, s6
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s6, 2
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s7
+; GFX90A-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v1, s6, v2
; GFX90A-NOTTGSPLIT-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen glc slc
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
@@ -414,13 +418,15 @@ define amdgpu_kernel void @private_nontemporal_load_1(
; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
-; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x0
+; GFX90A-TGSPLIT-NEXT: s_load_dword s7, s[8:9], 0x0
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
-; GFX90A-TGSPLIT-NEXT: s_mov_b32 s7, 0x3ff
-; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v1, v1, s7
-; GFX90A-TGSPLIT-NEXT: v_lshl_add_u32 v1, v1, 2, s6
+; GFX90A-TGSPLIT-NEXT: s_mov_b32 s6, 0x3ff
+; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v1, v1, s6
+; GFX90A-TGSPLIT-NEXT: s_mov_b32 s6, 2
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v2, s7
+; GFX90A-TGSPLIT-NEXT: v_lshl_add_u32 v1, v1, s6, v2
; GFX90A-TGSPLIT-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen glc slc
; GFX90A-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX90A-TGSPLIT-NEXT: global_store_dword v0, v1, s[4:5]
@@ -432,13 +438,15 @@ define amdgpu_kernel void @private_nontemporal_load_1(
; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX942-NOTTGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s3, 0x3ff
-; GFX942-NOTTGSPLIT-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX942-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s2, 0x3ff
+; GFX942-NOTTGSPLIT-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s2, 2
+; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v1, s2, v2
; GFX942-NOTTGSPLIT-NEXT: scratch_load_dword v1, v1, off nt
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-NOTTGSPLIT-NEXT: global_store_dword v0, v1, s[0:1]
@@ -450,13 +458,15 @@ define amdgpu_kernel void @private_nontemporal_load_1(
; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x0
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
-; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x0
+; GFX942-TGSPLIT-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x8
; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, 0
-; GFX942-TGSPLIT-NEXT: s_mov_b32 s3, 0x3ff
-; GFX942-TGSPLIT-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX942-TGSPLIT-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX942-TGSPLIT-NEXT: s_mov_b32 s2, 0x3ff
+; GFX942-TGSPLIT-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX942-TGSPLIT-NEXT: s_mov_b32 s2, 2
+; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v2, s3
+; GFX942-TGSPLIT-NEXT: v_lshl_add_u32 v1, v1, s2, v2
; GFX942-TGSPLIT-NEXT: scratch_load_dword v1, v1, off nt
; GFX942-TGSPLIT-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX942-TGSPLIT-NEXT: global_store_dword v0, v1, s[0:1]
@@ -468,13 +478,14 @@ define amdgpu_kernel void @private_nontemporal_load_1(
; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
-; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x0
+; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-WGP-NEXT: s_mov_b32 s3, 0x3ff
-; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff
+; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX11-WGP-NEXT: s_mov_b32 s2, 2
+; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX11-WGP-NEXT: scratch_load_b32 v1, v1, off slc dlc
; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -486,13 +497,14 @@ define amdgpu_kernel void @private_nontemporal_load_1(
; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
-; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x0
+; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-CU-NEXT: s_mov_b32 s3, 0x3ff
-; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX11-CU-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff
+; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX11-CU-NEXT: s_mov_b32 s2, 2
+; GFX11-CU-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX11-CU-NEXT: scratch_load_b32 v1, v1, off slc dlc
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -842,11 +854,13 @@ define amdgpu_kernel void @private_nontemporal_store_1(
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
-; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x8
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v0, 2, s5
+; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x8
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX10-WGP-NEXT: s_mov_b32 s5, 2
+; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v0, s5, s6
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
; GFX10-WGP-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
@@ -859,11 +873,13 @@ define amdgpu_kernel void @private_nontemporal_store_1(
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
-; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x8
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX10-CU-NEXT: v_lshl_add_u32 v1, v0, 2, s5
+; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x8
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX10-CU-NEXT: s_mov_b32 s5, 2
+; GFX10-CU-NEXT: v_lshl_add_u32 v1, v0, s5, s6
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
; GFX10-CU-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
@@ -899,13 +915,16 @@ define amdgpu_kernel void @private_nontemporal_store_1(
; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x8
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s6, 0x3ff
-; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s6
-; GFX90A-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v0, 2, s5
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
+; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-NOTTGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s5, 0x3ff
+; GFX90A-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s5
+; GFX90A-NOTTGSPLIT-NEXT: s_mov_b32 s5, 2
+; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v0, s5, v1
; GFX90A-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s4
; GFX90A-NOTTGSPLIT-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
@@ -918,13 +937,16 @@ define amdgpu_kernel void @private_nontemporal_store_1(
; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
-; GFX90A-TGSPLIT-NEXT: s_load_dword s5, s[8:9], 0x8
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX90A-TGSPLIT-NEXT: s_mov_b32 s6, 0x3ff
-; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s6
-; GFX90A-TGSPLIT-NEXT: v_lshl_add_u32 v1, v0, 2, s5
+; GFX90A-TGSPLIT-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX90A-TGSPLIT-NEXT: s_load_dword s6, s[8:9], 0x8
+; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX90A-TGSPLIT-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX90A-TGSPLIT-NEXT: s_mov_b32 s5, 0x3ff
+; GFX90A-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s5
+; GFX90A-TGSPLIT-NEXT: s_mov_b32 s5, 2
+; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v1, s6
+; GFX90A-TGSPLIT-NEXT: v_lshl_add_u32 v1, v0, s5, v1
; GFX90A-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX90A-TGSPLIT-NEXT: v_mov_b32_e32 v0, s4
; GFX90A-TGSPLIT-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen glc slc
@@ -935,13 +957,16 @@ define amdgpu_kernel void @private_nontemporal_store_1(
; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
-; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX942-NOTTGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x8
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s2, 0x3ff
-; GFX942-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX942-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v0, 2, s1
+; GFX942-NOTTGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-NOTTGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
+; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-NOTTGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x0
+; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s1, 0x3ff
+; GFX942-NOTTGSPLIT-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX942-NOTTGSPLIT-NEXT: s_mov_b32 s1, 2
+; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-NOTTGSPLIT-NEXT: v_lshl_add_u32 v1, v0, s1, v1
; GFX942-NOTTGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-NOTTGSPLIT-NEXT: v_mov_b32_e32 v0, s0
; GFX942-NOTTGSPLIT-NEXT: scratch_store_dword v1, v0, off nt
@@ -952,13 +977,16 @@ define amdgpu_kernel void @private_nontemporal_store_1(
; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[4:5], 0x8
-; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[2:3], s[4:5], 0x0
-; GFX942-TGSPLIT-NEXT: s_load_dword s1, s[4:5], 0x8
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[2:3], 0x0
-; GFX942-TGSPLIT-NEXT: s_mov_b32 s2, 0x3ff
-; GFX942-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX942-TGSPLIT-NEXT: v_lshl_add_u32 v1, v0, 2, s1
+; GFX942-TGSPLIT-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; GFX942-TGSPLIT-NEXT: s_load_dword s2, s[4:5], 0x8
+; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
+; GFX942-TGSPLIT-NEXT: s_load_dword s0, s[0:1], 0x0
+; GFX942-TGSPLIT-NEXT: s_mov_b32 s1, 0x3ff
+; GFX942-TGSPLIT-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX942-TGSPLIT-NEXT: s_mov_b32 s1, 2
+; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v1, s2
+; GFX942-TGSPLIT-NEXT: v_lshl_add_u32 v1, v0, s1, v1
; GFX942-TGSPLIT-NEXT: s_waitcnt lgkmcnt(0)
; GFX942-TGSPLIT-NEXT: v_mov_b32_e32 v0, s0
; GFX942-TGSPLIT-NEXT: scratch_store_dword v1, v0, off nt
@@ -969,13 +997,15 @@ define amdgpu_kernel void @private_nontemporal_store_1(
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
-; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x8
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff
-; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v0, 2, s1
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX11-WGP-NEXT: s_mov_b32 s1, 0x3ff
+; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX11-WGP-NEXT: s_mov_b32 s1, 2
+; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v0, s1, s2
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
; GFX11-WGP-NEXT: scratch_store_b32 v1, v0, off glc slc dlc
@@ -986,13 +1016,15 @@ define amdgpu_kernel void @private_nontemporal_store_1(
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
-; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x8
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff
-; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX11-CU-NEXT: v_lshl_add_u32 v1, v0, 2, s1
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX11-CU-NEXT: s_mov_b32 s1, 0x3ff
+; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX11-CU-NEXT: s_mov_b32 s1, 2
+; GFX11-CU-NEXT: v_lshl_add_u32 v1, v0, s1, s2
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
; GFX11-CU-NEXT: scratch_store_b32 v1, v0, off glc slc dlc
diff --git a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
index 903d5ab3455ca..fd822659c3439 100644
--- a/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
+++ b/llvm/test/CodeGen/AMDGPU/memory-legalizer-private-volatile.ll
@@ -272,11 +272,12 @@ define amdgpu_kernel void @private_volatile_load_1(
; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
-; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x0
+; GFX10-WGP-NEXT: s_load_dword s7, s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-WGP-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v1, 2, s6
+; GFX10-WGP-NEXT: s_mov_b32 s6, 2
+; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v1, s6, s7
; GFX10-WGP-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen glc dlc
; GFX10-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-WGP-NEXT: global_store_dword v0, v1, s[4:5]
@@ -290,11 +291,12 @@ define amdgpu_kernel void @private_volatile_load_1(
; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
-; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x0
+; GFX10-CU-NEXT: s_load_dword s7, s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x8
; GFX10-CU-NEXT: v_mov_b32_e32 v0, 0
-; GFX10-CU-NEXT: v_lshl_add_u32 v1, v1, 2, s6
+; GFX10-CU-NEXT: s_mov_b32 s6, 2
+; GFX10-CU-NEXT: v_lshl_add_u32 v1, v1, s6, s7
; GFX10-CU-NEXT: buffer_load_dword v1, v1, s[0:3], 0 offen glc dlc
; GFX10-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX10-CU-NEXT: global_store_dword v0, v1, s[4:5]
@@ -337,13 +339,14 @@ define amdgpu_kernel void @private_volatile_load_1(
; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
-; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x0
+; GFX11-WGP-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX11-WGP-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-WGP-NEXT: s_mov_b32 s3, 0x3ff
-; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff
+; GFX11-WGP-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX11-WGP-NEXT: s_mov_b32 s2, 2
+; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX11-WGP-NEXT: scratch_load_b32 v1, v1, off glc dlc
; GFX11-WGP-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-WGP-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -355,13 +358,14 @@ define amdgpu_kernel void @private_volatile_load_1(
; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
-; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x0
+; GFX11-CU-NEXT: s_load_b32 s3, s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX11-CU-NEXT: v_mov_b32_e32 v0, 0
-; GFX11-CU-NEXT: s_mov_b32 s3, 0x3ff
-; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s3
-; GFX11-CU-NEXT: v_lshl_add_u32 v1, v1, 2, s2
+; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff
+; GFX11-CU-NEXT: v_and_b32_e64 v1, v1, s2
+; GFX11-CU-NEXT: s_mov_b32 s2, 2
+; GFX11-CU-NEXT: v_lshl_add_u32 v1, v1, s2, s3
; GFX11-CU-NEXT: scratch_load_b32 v1, v1, off glc dlc
; GFX11-CU-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
; GFX11-CU-NEXT: global_store_b32 v0, v1, s[0:1]
@@ -685,11 +689,13 @@ define amdgpu_kernel void @private_volatile_store_1(
; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX10-WGP-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
-; GFX10-WGP-NEXT: s_load_dword s5, s[8:9], 0x8
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-WGP-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v0, 2, s5
+; GFX10-WGP-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-WGP-NEXT: s_load_dword s6, s[8:9], 0x8
+; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-WGP-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX10-WGP-NEXT: s_mov_b32 s5, 2
+; GFX10-WGP-NEXT: v_lshl_add_u32 v1, v0, s5, s6
; GFX10-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-WGP-NEXT: v_mov_b32_e32 v0, s4
; GFX10-WGP-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
@@ -703,11 +709,13 @@ define amdgpu_kernel void @private_volatile_store_1(
; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: s_load_dword s4, s[8:9], 0x8
-; GFX10-CU-NEXT: s_load_dwordx2 s[6:7], s[8:9], 0x0
-; GFX10-CU-NEXT: s_load_dword s5, s[8:9], 0x8
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-CU-NEXT: s_load_dword s4, s[6:7], 0x0
-; GFX10-CU-NEXT: v_lshl_add_u32 v1, v0, 2, s5
+; GFX10-CU-NEXT: s_load_dwordx2 s[4:5], s[8:9], 0x0
+; GFX10-CU-NEXT: s_load_dword s6, s[8:9], 0x8
+; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-CU-NEXT: s_load_dword s4, s[4:5], 0x0
+; GFX10-CU-NEXT: s_mov_b32 s5, 2
+; GFX10-CU-NEXT: v_lshl_add_u32 v1, v0, s5, s6
; GFX10-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-CU-NEXT: v_mov_b32_e32 v0, s4
; GFX10-CU-NEXT: buffer_store_dword v0, v1, s[0:3], 0 offen
@@ -743,13 +751,15 @@ define amdgpu_kernel void @private_volatile_store_1(
; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-WGP-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
-; GFX11-WGP-NEXT: s_load_b32 s1, s[4:5], 0x8
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-WGP-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-WGP-NEXT: s_mov_b32 s2, 0x3ff
-; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v0, 2, s1
+; GFX11-WGP-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-WGP-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-WGP-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX11-WGP-NEXT: s_mov_b32 s1, 0x3ff
+; GFX11-WGP-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX11-WGP-NEXT: s_mov_b32 s1, 2
+; GFX11-WGP-NEXT: v_lshl_add_u32 v1, v0, s1, s2
; GFX11-WGP-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-WGP-NEXT: v_mov_b32_e32 v0, s0
; GFX11-WGP-NEXT: scratch_store_b32 v1, v0, off dlc
@@ -761,13 +771,15 @@ define amdgpu_kernel void @private_volatile_store_1(
; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: s_load_b32 s0, s[4:5], 0x8
-; GFX11-CU-NEXT: s_load_b64 s[2:3], s[4:5], 0x0
-; GFX11-CU-NEXT: s_load_b32 s1, s[4:5], 0x8
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-CU-NEXT: s_load_b32 s0, s[2:3], 0x0
-; GFX11-CU-NEXT: s_mov_b32 s2, 0x3ff
-; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s2
-; GFX11-CU-NEXT: v_lshl_add_u32 v1, v0, 2, s1
+; GFX11-CU-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX11-CU-NEXT: s_load_b32 s2, s[4:5], 0x8
+; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
+; GFX11-CU-NEXT: s_load_b32 s0, s[0:1], 0x0
+; GFX11-CU-NEXT: s_mov_b32 s1, 0x3ff
+; GFX11-CU-NEXT: v_and_b32_e64 v0, v0, s1
+; GFX11-CU-NEXT: s_mov_b32 s1, 2
+; GFX11-CU-NEXT: v_lshl_add_u32 v1, v0, s1, s2
; GFX11-CU-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-CU-NEXT: v_mov_b32_e32 v0, s0
; GFX11-CU-NEXT: scratch_store_b32 v1, v0, off dlc
diff --git a/llvm/test/CodeGen/AMDGPU/shl_add.ll b/llvm/test/CodeGen/AMDGPU/shl_add.ll
index 7af6c8b07974f..aaa34f4b47909 100644
--- a/llvm/test/CodeGen/AMDGPU/shl_add.ll
+++ b/llvm/test/CodeGen/AMDGPU/shl_add.ll
@@ -3,6 +3,7 @@
; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -mcpu=gfx900 | FileCheck -check-prefix=GFX9 %s
; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1010 | FileCheck -check-prefix=GFX10 %s
; RUN: llc < %s -mtriple=amdgcn-amd-mesa3d -mcpu=gfx1100 -amdgpu-enable-delay-alu=0 | FileCheck -check-prefix=GFX10 %s
+; RUN: llc -O0 < %s -mtriple=amdgcn-amd-mesa3d -mcpu=gfx900 | FileCheck -check-prefix=GFX9-O0 %s
; ===================================================================================
; V_LSHL_ADD_U32
@@ -24,6 +25,11 @@ define amdgpu_ps float @shl_add(i32 %a, i32 %b, i32 %c) {
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshl_add_u32 v0, v0, v1, v2
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX9-O0-LABEL: shl_add:
+; GFX9-O0: ; %bb.0:
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, v1, v2
+; GFX9-O0-NEXT: ; return to shader part epilog
%x = shl i32 %a, %b
%result = add i32 %x, %c
%bc = bitcast i32 %result to float
@@ -48,6 +54,12 @@ define amdgpu_ps float @shl_add_vgpr_a(i32 %a, i32 inreg %b, i32 inreg %c) {
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshl_add_u32 v0, v0, s2, s3
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX9-O0-LABEL: shl_add_vgpr_a:
+; GFX9-O0: ; %bb.0:
+; GFX9-O0-NEXT: v_lshlrev_b32_e64 v0, s2, v0
+; GFX9-O0-NEXT: v_add_u32_e64 v0, v0, s3
+; GFX9-O0-NEXT: ; return to shader part epilog
%x = shl i32 %a, %b
%result = add i32 %x, %c
%bc = bitcast i32 %result to float
@@ -70,6 +82,11 @@ define amdgpu_ps float @shl_add_vgpr_all(i32 %a, i32 %b, i32 %c) {
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshl_add_u32 v0, v0, v1, v2
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX9-O0-LABEL: shl_add_vgpr_all:
+; GFX9-O0: ; %bb.0:
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, v1, v2
+; GFX9-O0-NEXT: ; return to shader part epilog
%x = shl i32 %a, %b
%result = add i32 %x, %c
%bc = bitcast i32 %result to float
@@ -92,6 +109,11 @@ define amdgpu_ps float @shl_add_vgpr_ab(i32 %a, i32 %b, i32 inreg %c) {
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshl_add_u32 v0, v0, v1, s2
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX9-O0-LABEL: shl_add_vgpr_ab:
+; GFX9-O0: ; %bb.0:
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, v1, s2
+; GFX9-O0-NEXT: ; return to shader part epilog
%x = shl i32 %a, %b
%result = add i32 %x, %c
%bc = bitcast i32 %result to float
@@ -114,8 +136,219 @@ define amdgpu_ps float @shl_add_vgpr_const(i32 %a, i32 %b) {
; GFX10: ; %bb.0:
; GFX10-NEXT: v_lshl_add_u32 v0, v0, 3, v1
; GFX10-NEXT: ; return to shader part epilog
+;
+; GFX9-O0-LABEL: shl_add_vgpr_const:
+; GFX9-O0: ; %bb.0:
+; GFX9-O0-NEXT: s_mov_b32 s0, 3
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s0, v1
+; GFX9-O0-NEXT: ; return to shader part epilog
%x = shl i32 %a, 3
%result = add i32 %x, %b
%bc = bitcast i32 %result to float
ret float %bc
}
+
+; ===================================================================================
+; Divergent shlN_add chains at -O0 must fall through to V_LSHL_ADD_U32 rather
+; than selecting the SALU s_lshl[1-4]_add_u32 patterns.
+; ===================================================================================
+
+define i32 @v_shl1_add_chain(i32 %src) {
+; VI-LABEL: v_shl1_add_chain:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v0, 5, v0
+; VI-NEXT: v_add_u32_e32 v0, vcc, 0x9b3e1f00, v0
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_shl1_add_chain:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x9b3e1f00
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 5, v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_shl1_add_chain:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshl_add_u32 v0, v0, 5, 0x9b3e1f00
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-O0-LABEL: v_shl1_add_chain:
+; GFX9-O0: ; %bb.0:
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-O0-NEXT: s_mov_b32 s5, 0x5020100
+; GFX9-O0-NEXT: s_mov_b32 s4, 1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: s_setpc_b64 s[30:31]
+ %s1 = shl i32 %src, 1
+ %a1 = add i32 %s1, 84017408
+ %s2 = shl i32 %a1, 1
+ %a2 = add i32 %s2, 84017408
+ %s3 = shl i32 %a2, 1
+ %a3 = add i32 %s3, 84017408
+ %s4 = shl i32 %a3, 1
+ %a4 = add i32 %s4, 84017408
+ %s5 = shl i32 %a4, 1
+ %a5 = add i32 %s5, 84017408
+ ret i32 %a5
+}
+
+define i32 @v_shl2_add_chain(i32 %src) {
+; VI-LABEL: v_shl2_add_chain:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v0, 10, v0
+; VI-NEXT: v_add_u32_e32 v0, vcc, 0xabab5500, v0
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_shl2_add_chain:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, 0xabab5500
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 10, v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_shl2_add_chain:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshl_add_u32 v0, v0, 10, 0xabab5500
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-O0-LABEL: v_shl2_add_chain:
+; GFX9-O0: ; %bb.0:
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-O0-NEXT: s_mov_b32 s5, 0x5020100
+; GFX9-O0-NEXT: s_mov_b32 s4, 2
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: s_setpc_b64 s[30:31]
+ %s1 = shl i32 %src, 2
+ %a1 = add i32 %s1, 84017408
+ %s2 = shl i32 %a1, 2
+ %a2 = add i32 %s2, 84017408
+ %s3 = shl i32 %a2, 2
+ %a3 = add i32 %s3, 84017408
+ %s4 = shl i32 %a3, 2
+ %a4 = add i32 %s4, 84017408
+ %s5 = shl i32 %a4, 2
+ %a5 = add i32 %s5, 84017408
+ ret i32 %a5
+}
+
+define i32 @v_shl3_add_chain(i32 %src) {
+; VI-LABEL: v_shl3_add_chain:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v0, 15, v0
+; VI-NEXT: v_add_u32_e32 v0, vcc, 0x91a44900, v0
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_shl3_add_chain:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x91a44900
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 15, v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_shl3_add_chain:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshl_add_u32 v0, v0, 15, 0x91a44900
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-O0-LABEL: v_shl3_add_chain:
+; GFX9-O0: ; %bb.0:
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-O0-NEXT: s_mov_b32 s5, 0x5020100
+; GFX9-O0-NEXT: s_mov_b32 s4, 3
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: s_setpc_b64 s[30:31]
+ %s1 = shl i32 %src, 3
+ %a1 = add i32 %s1, 84017408
+ %s2 = shl i32 %a1, 3
+ %a2 = add i32 %s2, 84017408
+ %s3 = shl i32 %a2, 3
+ %a3 = add i32 %s3, 84017408
+ %s4 = shl i32 %a3, 3
+ %a4 = add i32 %s4, 84017408
+ %s5 = shl i32 %a4, 3
+ %a5 = add i32 %s5, 84017408
+ ret i32 %a5
+}
+
+define i32 @v_shl4_add_chain(i32 %src) {
+; VI-LABEL: v_shl4_add_chain:
+; VI: ; %bb.0:
+; VI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; VI-NEXT: v_lshlrev_b32_e32 v0, 20, v0
+; VI-NEXT: v_add_u32_e32 v0, vcc, 0x78331100, v0
+; VI-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-LABEL: v_shl4_add_chain:
+; GFX9: ; %bb.0:
+; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-NEXT: v_mov_b32_e32 v1, 0x78331100
+; GFX9-NEXT: v_lshl_add_u32 v0, v0, 20, v1
+; GFX9-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX10-LABEL: v_shl4_add_chain:
+; GFX10: ; %bb.0:
+; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX10-NEXT: v_lshl_add_u32 v0, v0, 20, 0x78331100
+; GFX10-NEXT: s_setpc_b64 s[30:31]
+;
+; GFX9-O0-LABEL: v_shl4_add_chain:
+; GFX9-O0: ; %bb.0:
+; GFX9-O0-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX9-O0-NEXT: s_mov_b32 s5, 0x5020100
+; GFX9-O0-NEXT: s_mov_b32 s4, 4
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: v_mov_b32_e32 v1, s5
+; GFX9-O0-NEXT: v_lshl_add_u32 v0, v0, s4, v1
+; GFX9-O0-NEXT: s_setpc_b64 s[30:31]
+ %s1 = shl i32 %src, 4
+ %a1 = add i32 %s1, 84017408
+ %s2 = shl i32 %a1, 4
+ %a2 = add i32 %s2, 84017408
+ %s3 = shl i32 %a2, 4
+ %a3 = add i32 %s3, 84017408
+ %s4 = shl i32 %a3, 4
+ %a4 = add i32 %s4, 84017408
+ %s5 = shl i32 %a4, 4
+ %a5 = add i32 %s5, 84017408
+ ret i32 %a5
+}
diff --git a/llvm/test/CodeGen/AMDGPU/wqm.ll b/llvm/test/CodeGen/AMDGPU/wqm.ll
index 4e5b99ae7447d..219a46f37f415 100644
--- a/llvm/test/CodeGen/AMDGPU/wqm.ll
+++ b/llvm/test/CodeGen/AMDGPU/wqm.ll
@@ -2029,10 +2029,10 @@ define amdgpu_ps void @test_alloca(float %data, i32 %a, i32 %idx) nounwind {
; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1]
; GFX9-W64-NEXT: buffer_store_dword v0, off, s[0:3], 0
; GFX9-W64-NEXT: s_wqm_b64 exec, exec
-; GFX9-W64-NEXT: s_mov_b32 s2, 0
; GFX9-W64-NEXT: buffer_store_dword v1, off, s[8:11], 0
; GFX9-W64-NEXT: s_waitcnt vmcnt(0)
-; GFX9-W64-NEXT: v_lshl_add_u32 v1, v2, 2, s2
+; GFX9-W64-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-W64-NEXT: v_lshl_add_u32 v1, v2, 2, v1
; GFX9-W64-NEXT: buffer_load_dword v1, v1, s[8:11], 0 offen
; GFX9-W64-NEXT: s_and_b64 exec, exec, s[0:1]
; GFX9-W64-NEXT: s_waitcnt vmcnt(0)
More information about the llvm-commits
mailing list