[llvm] [SDag] Notify listeners when deleting a node (PR #66991)
Sergei Barannikov via llvm-commits
llvm-commits at lists.llvm.org
Tue Oct 6 12:50:29 PDT 2026
https://github.com/s-barannikov updated https://github.com/llvm/llvm-project/pull/66991
>From d239e6aa3a89d4fb860ad5feb12fda1e64feb5c8 Mon Sep 17 00:00:00 2001
From: Sergei Barannikov <barannikov88 at gmail.com>
Date: Thu, 21 Sep 2023 11:54:39 +0300
Subject: [PATCH] [SDag] Notify listeners when deleting a node
SelectionDAG::DeleteNode does not notify listeners about node deletion.
As a result, SelectionDAG::Legalize may skip legalization of some nodes
resulting in "Legalized selection DAG" containing illegal nodes. These
nodes will be legalized during subsequent DAG combining phase, but this
may be too late for some patterns to match.
---
.../lib/CodeGen/SelectionDAG/SelectionDAG.cpp | 3 +
llvm/test/CodeGen/AMDGPU/half.ll | 148 +-
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 161 +-
llvm/test/CodeGen/AMDGPU/load-constant-i16.ll | 193 +-
llvm/test/CodeGen/AMDGPU/load-constant-i8.ll | 148 +-
llvm/test/CodeGen/AMDGPU/load-global-i16.ll | 160 +-
llvm/test/CodeGen/AMDGPU/load-global-i8.ll | 224 +-
llvm/test/CodeGen/AMDGPU/load-local-i16.ll | 774 +++--
.../CodeGen/AMDGPU/memintrinsic-unroll.ll | 2563 ++++++++---------
.../CodeGen/X86/buildvec-strided-loads.ll | 2 +-
.../X86/masked_gather_scatter_widen.ll | 3 +-
llvm/test/CodeGen/X86/pr40730.ll | 16 +-
.../vector-interleaved-load-i16-stride-5.ll | 8 +-
.../vector-interleaved-store-i16-stride-8.ll | 612 ++--
.../CodeGen/X86/vector-shuffle-256-v16.ll | 24 +-
.../zero_extend_vector_inreg_of_broadcast.ll | 106 +-
16 files changed, 2558 insertions(+), 2587 deletions(-)
diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
index 77922d8fb13442..20c8f025f035e5 100644
--- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
+++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp
@@ -1162,6 +1162,9 @@ void SelectionDAG::RemoveDeadNode(SDNode *N){
}
void SelectionDAG::DeleteNode(SDNode *N) {
+ for (DAGUpdateListener *DUL = UpdateListeners; DUL; DUL = DUL->Next)
+ DUL->NodeDeleted(N, nullptr);
+
// First take this out of the appropriate CSE map.
RemoveNodeFromCSEMaps(N);
diff --git a/llvm/test/CodeGen/AMDGPU/half.ll b/llvm/test/CodeGen/AMDGPU/half.ll
index 3f96fc768ed1e1..a6dacfa356464e 100644
--- a/llvm/test/CodeGen/AMDGPU/half.ll
+++ b/llvm/test/CodeGen/AMDGPU/half.ll
@@ -2297,7 +2297,7 @@ define amdgpu_kernel void @global_extload_v16f16_to_v16f64(ptr addrspace(1) %out
; VI-NEXT: v_mov_b32_e32 v17, s2
; VI-NEXT: v_mov_b32_e32 v12, s1
; VI-NEXT: v_mov_b32_e32 v11, s0
-; VI-NEXT: s_add_u32 s2, s0, 0x50
+; VI-NEXT: s_add_u32 s2, s0, 0x70
; VI-NEXT: s_addc_u32 s3, s1, 0
; VI-NEXT: s_waitcnt vmcnt(1)
; VI-NEXT: v_cvt_f32_f16_e32 v8, v7
@@ -2309,11 +2309,11 @@ define amdgpu_kernel void @global_extload_v16f16_to_v16f64(ptr addrspace(1) %out
; VI-NEXT: v_cvt_f32_f16_e32 v7, v6
; VI-NEXT: v_cvt_f32_f16_sdwa v8, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; VI-NEXT: s_waitcnt vmcnt(1)
-; VI-NEXT: v_cvt_f32_f16_e32 v10, v2
+; VI-NEXT: v_cvt_f32_f16_e32 v10, v0
; VI-NEXT: v_mov_b32_e32 v13, s2
; VI-NEXT: v_cvt_f64_f32_e32 v[6:7], v7
; VI-NEXT: v_cvt_f64_f32_e32 v[8:9], v8
-; VI-NEXT: s_add_u32 s2, s0, 64
+; VI-NEXT: s_add_u32 s2, s0, 0x60
; VI-NEXT: s_addc_u32 s3, s1, 0
; VI-NEXT: flat_store_dwordx4 v[15:16], v[6:9]
; VI-NEXT: v_mov_b32_e32 v16, s3
@@ -2324,37 +2324,37 @@ define amdgpu_kernel void @global_extload_v16f16_to_v16f64(ptr addrspace(1) %out
; VI-NEXT: v_cvt_f64_f32_e32 v[4:5], v6
; VI-NEXT: v_cvt_f64_f32_e32 v[6:7], v7
; VI-NEXT: v_mov_b32_e32 v15, s2
-; VI-NEXT: s_add_u32 s2, s0, 0x70
+; VI-NEXT: s_add_u32 s2, s0, 0x50
; VI-NEXT: s_addc_u32 s3, s1, 0
; VI-NEXT: flat_store_dwordx4 v[17:18], v[4:7]
-; VI-NEXT: v_cvt_f32_f16_sdwa v17, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-NEXT: v_cvt_f32_f16_sdwa v17, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
; VI-NEXT: v_cvt_f64_f32_e32 v[4:5], v8
; VI-NEXT: v_cvt_f64_f32_e32 v[6:7], v9
-; VI-NEXT: v_cvt_f32_f16_sdwa v9, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_sdwa v8, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v2, v1
+; VI-NEXT: v_cvt_f32_f16_sdwa v8, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-NEXT: v_cvt_f32_f16_e32 v0, v3
+; VI-NEXT: s_add_u32 s0, s0, 64
; VI-NEXT: flat_store_dwordx4 v[11:12], v[4:7]
-; VI-NEXT: v_cvt_f32_f16_sdwa v11, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
-; VI-NEXT: v_cvt_f32_f16_e32 v7, v3
-; VI-NEXT: v_cvt_f64_f32_e32 v[3:4], v9
-; VI-NEXT: v_cvt_f32_f16_e32 v9, v0
-; VI-NEXT: v_cvt_f64_f32_e32 v[1:2], v2
-; VI-NEXT: v_cvt_f64_f32_e32 v[5:6], v10
-; VI-NEXT: v_cvt_f64_f32_e32 v[11:12], v11
-; VI-NEXT: v_cvt_f64_f32_e32 v[9:10], v9
-; VI-NEXT: flat_store_dwordx4 v[13:14], v[1:4]
-; VI-NEXT: s_add_u32 s0, s0, 0x60
-; VI-NEXT: v_cvt_f64_f32_e32 v[0:1], v7
-; VI-NEXT: v_cvt_f64_f32_e32 v[2:3], v17
-; VI-NEXT: v_cvt_f64_f32_e32 v[7:8], v8
+; VI-NEXT: v_cvt_f32_f16_sdwa v12, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-NEXT: v_cvt_f64_f32_e32 v[5:6], v8
+; VI-NEXT: v_cvt_f32_f16_e32 v8, v2
+; VI-NEXT: v_cvt_f32_f16_sdwa v2, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1
+; VI-NEXT: v_cvt_f32_f16_e32 v7, v1
+; VI-NEXT: v_cvt_f64_f32_e32 v[3:4], v0
+; VI-NEXT: v_cvt_f64_f32_e32 v[0:1], v10
+; VI-NEXT: v_cvt_f64_f32_e32 v[8:9], v8
+; VI-NEXT: v_cvt_f64_f32_e32 v[10:11], v2
+; VI-NEXT: flat_store_dwordx4 v[13:14], v[3:6]
; VI-NEXT: s_addc_u32 s1, s1, 0
+; VI-NEXT: v_cvt_f64_f32_e32 v[4:5], v7
+; VI-NEXT: v_cvt_f64_f32_e32 v[6:7], v12
+; VI-NEXT: v_cvt_f64_f32_e32 v[2:3], v17
; VI-NEXT: v_mov_b32_e32 v20, s3
; VI-NEXT: v_mov_b32_e32 v19, s2
-; VI-NEXT: v_mov_b32_e32 v14, s1
-; VI-NEXT: v_mov_b32_e32 v13, s0
-; VI-NEXT: flat_store_dwordx4 v[15:16], v[9:12]
-; VI-NEXT: flat_store_dwordx4 v[19:20], v[0:3]
-; VI-NEXT: flat_store_dwordx4 v[13:14], v[5:8]
+; VI-NEXT: v_mov_b32_e32 v13, s1
+; VI-NEXT: v_mov_b32_e32 v12, s0
+; VI-NEXT: flat_store_dwordx4 v[15:16], v[8:11]
+; VI-NEXT: flat_store_dwordx4 v[19:20], v[4:7]
+; VI-NEXT: flat_store_dwordx4 v[12:13], v[0:3]
; VI-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: global_extload_v16f16_to_v16f64:
@@ -2366,40 +2366,40 @@ define amdgpu_kernel void @global_extload_v16f16_to_v16f64(ptr addrspace(1) %out
; GFX11-TRUE16-NEXT: global_load_b128 v[0:3], v32, s[2:3]
; GFX11-TRUE16-NEXT: global_load_b128 v[4:7], v32, s[2:3] offset:16
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v10, v1.l
-; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v23, 16, v5
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v4
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v15, v7.l
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v23, v7.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v14, v6.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v22, v6.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v19, 16, v5
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v10, v1.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v15, 16, v4
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v13, v3.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v12, v2.l
; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v18, v4.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v22, v5.l
-; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[4:5], v10
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v10, v23.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v34, v11.l
-; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v11, v19.l
-; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v7, v7.l
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v6, v6.l
+; GFX11-TRUE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v18, v5.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v34, v11.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v11, v19.l
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v14, v4.l
+; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[4:5], v10
+; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v10, v15.l
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v8, v0.l
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v3, v3.l
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v2, v2.l
-; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[28:29], v22
-; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[30:31], v10
-; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[24:25], v18
-; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[26:27], v11
+; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[28:29], v23
+; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[30:31], v7
+; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[24:25], v22
+; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[26:27], v6
; GFX11-TRUE16-NEXT: v_cvt_f32_f16_e32 v33, v9.l
-; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[20:21], v15
-; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[22:23], v7
+; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[20:21], v18
+; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[22:23], v11
; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[16:17], v14
-; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[18:19], v6
+; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[18:19], v10
; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v8
; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[8:9], v12
; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[12:13], v13
@@ -2408,10 +2408,10 @@ define amdgpu_kernel void @global_extload_v16f16_to_v16f64(ptr addrspace(1) %out
; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[6:7], v34
; GFX11-TRUE16-NEXT: v_cvt_f64_f32_e32 v[2:3], v33
; GFX11-TRUE16-NEXT: s_clause 0x7
-; GFX11-TRUE16-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:80
-; GFX11-TRUE16-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:64
-; GFX11-TRUE16-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:112
-; GFX11-TRUE16-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:96
+; GFX11-TRUE16-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:112
+; GFX11-TRUE16-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:96
+; GFX11-TRUE16-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:80
+; GFX11-TRUE16-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:64
; GFX11-TRUE16-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:48
; GFX11-TRUE16-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:32
; GFX11-TRUE16-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:16
@@ -2427,40 +2427,40 @@ define amdgpu_kernel void @global_extload_v16f16_to_v16f64(ptr addrspace(1) %out
; GFX11-FAKE16-NEXT: global_load_b128 v[0:3], v32, s[2:3]
; GFX11-FAKE16-NEXT: global_load_b128 v[4:7], v32, s[2:3] offset:16
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v10, v1
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v23, 16, v5
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v11, 16, v1
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v19, 16, v4
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v15, v7
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v23, v7
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v14, v6
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v22, v6
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v6
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v19, 16, v5
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v10, v1
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v15, 16, v4
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v13, v3
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v12, v2
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v18, v4
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v22, v5
-; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[4:5], v10
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v10, v23
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v34, v11
-; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v11, v19
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v7, v7
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v6, v6
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v18, v5
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v34, v11
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v11, v19
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v14, v4
+; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[4:5], v10
+; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v10, v15
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v8, v0
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v3, v3
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v2, v2
-; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[28:29], v22
-; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[30:31], v10
-; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[24:25], v18
-; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[26:27], v11
+; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[28:29], v23
+; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[30:31], v7
+; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[24:25], v22
+; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[26:27], v6
; GFX11-FAKE16-NEXT: v_cvt_f32_f16_e32 v33, v9
-; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[20:21], v15
-; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[22:23], v7
+; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[20:21], v18
+; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[22:23], v11
; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[16:17], v14
-; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[18:19], v6
+; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[18:19], v10
; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[0:1], v8
; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[8:9], v12
; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[12:13], v13
@@ -2469,10 +2469,10 @@ define amdgpu_kernel void @global_extload_v16f16_to_v16f64(ptr addrspace(1) %out
; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[6:7], v34
; GFX11-FAKE16-NEXT: v_cvt_f64_f32_e32 v[2:3], v33
; GFX11-FAKE16-NEXT: s_clause 0x7
-; GFX11-FAKE16-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:80
-; GFX11-FAKE16-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:64
-; GFX11-FAKE16-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:112
-; GFX11-FAKE16-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:96
+; GFX11-FAKE16-NEXT: global_store_b128 v32, v[28:31], s[0:1] offset:112
+; GFX11-FAKE16-NEXT: global_store_b128 v32, v[24:27], s[0:1] offset:96
+; GFX11-FAKE16-NEXT: global_store_b128 v32, v[20:23], s[0:1] offset:80
+; GFX11-FAKE16-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:64
; GFX11-FAKE16-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:48
; GFX11-FAKE16-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:32
; GFX11-FAKE16-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index 621d0dedb463fe..7aa5ec2b99e2e7 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -7075,33 +7075,33 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX6-NEXT: v_mov_b32_e32 v26, v1
; GFX6-NEXT: v_mov_b32_e32 v28, v1
; GFX6-NEXT: s_waitcnt vmcnt(0)
-; GFX6-NEXT: v_bfe_u32 v2, v29, 11, 1
-; GFX6-NEXT: v_bfe_u32 v0, v29, 10, 1
-; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:80
-; GFX6-NEXT: v_bfe_u32 v5, v29, 9, 1
+; GFX6-NEXT: v_lshrrev_b32_e32 v2, 15, v29
+; GFX6-NEXT: v_bfe_u32 v0, v29, 14, 1
+; GFX6-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112
+; GFX6-NEXT: v_bfe_u32 v5, v29, 13, 1
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_bfe_u32 v3, v29, 8, 1
-; GFX6-NEXT: buffer_store_dwordx4 v[3:6], off, s[0:3], 0 offset:64
-; GFX6-NEXT: v_lshrrev_b32_e32 v8, 15, v29
+; GFX6-NEXT: v_bfe_u32 v3, v29, 12, 1
+; GFX6-NEXT: buffer_store_dwordx4 v[3:6], off, s[0:3], 0 offset:96
+; GFX6-NEXT: v_bfe_u32 v8, v29, 11, 1
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_bfe_u32 v6, v29, 14, 1
-; GFX6-NEXT: buffer_store_dwordx4 v[6:9], off, s[0:3], 0 offset:112
-; GFX6-NEXT: v_bfe_u32 v11, v29, 13, 1
+; GFX6-NEXT: v_bfe_u32 v6, v29, 10, 1
+; GFX6-NEXT: buffer_store_dwordx4 v[6:9], off, s[0:3], 0 offset:80
+; GFX6-NEXT: v_bfe_u32 v11, v29, 9, 1
; GFX6-NEXT: s_waitcnt expcnt(0)
-; GFX6-NEXT: v_bfe_u32 v9, v29, 12, 1
-; GFX6-NEXT: v_bfe_u32 v27, v29, 5, 1
-; GFX6-NEXT: v_bfe_u32 v23, v29, 7, 1
-; GFX6-NEXT: v_bfe_u32 v19, v29, 1, 1
-; GFX6-NEXT: v_bfe_u32 v15, v29, 3, 1
-; GFX6-NEXT: v_bfe_u32 v25, v29, 4, 1
-; GFX6-NEXT: v_bfe_u32 v21, v29, 6, 1
-; GFX6-NEXT: v_and_b32_e32 v17, 1, v29
-; GFX6-NEXT: v_bfe_u32 v13, v29, 2, 1
-; GFX6-NEXT: buffer_store_dwordx4 v[9:12], off, s[0:3], 0 offset:96
-; GFX6-NEXT: buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:16
-; GFX6-NEXT: buffer_store_dwordx4 v[17:20], off, s[0:3], 0
-; GFX6-NEXT: buffer_store_dwordx4 v[21:24], off, s[0:3], 0 offset:48
-; GFX6-NEXT: buffer_store_dwordx4 v[25:28], off, s[0:3], 0 offset:32
+; GFX6-NEXT: v_bfe_u32 v9, v29, 8, 1
+; GFX6-NEXT: v_bfe_u32 v27, v29, 1, 1
+; GFX6-NEXT: v_bfe_u32 v23, v29, 3, 1
+; GFX6-NEXT: v_bfe_u32 v19, v29, 5, 1
+; GFX6-NEXT: v_bfe_u32 v15, v29, 7, 1
+; GFX6-NEXT: v_and_b32_e32 v25, 1, v29
+; GFX6-NEXT: v_bfe_u32 v21, v29, 2, 1
+; GFX6-NEXT: v_bfe_u32 v17, v29, 4, 1
+; GFX6-NEXT: v_bfe_u32 v13, v29, 6, 1
+; GFX6-NEXT: buffer_store_dwordx4 v[9:12], off, s[0:3], 0 offset:64
+; GFX6-NEXT: buffer_store_dwordx4 v[13:16], off, s[0:3], 0 offset:48
+; GFX6-NEXT: buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:32
+; GFX6-NEXT: buffer_store_dwordx4 v[21:24], off, s[0:3], 0 offset:16
+; GFX6-NEXT: buffer_store_dwordx4 v[25:28], off, s[0:3], 0
; GFX6-NEXT: s_endpgm
;
; GFX8-LABEL: constant_zextload_v16i1_to_v16i64:
@@ -7109,8 +7109,6 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX8-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x24
; GFX8-NEXT: v_mov_b32_e32 v3, 0
; GFX8-NEXT: v_mov_b32_e32 v5, v3
-; GFX8-NEXT: v_mov_b32_e32 v7, v3
-; GFX8-NEXT: v_mov_b32_e32 v9, v3
; GFX8-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NEXT: v_mov_b32_e32 v0, s2
; GFX8-NEXT: v_mov_b32_e32 v1, s3
@@ -7118,9 +7116,8 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX8-NEXT: v_mov_b32_e32 v1, v3
; GFX8-NEXT: s_waitcnt vmcnt(0)
; GFX8-NEXT: v_readfirstlane_b32 s2, v0
-; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX8-NEXT: s_bfe_u32 s4, s2, 0x10009
-; GFX8-NEXT: s_bfe_u32 s5, s2, 0x1000d
+; GFX8-NEXT: s_bfe_u32 s4, s2, 0x1000d
+; GFX8-NEXT: s_bfe_u32 s5, s2, 0x10009
; GFX8-NEXT: s_bfe_u32 s6, s2, 0x10007
; GFX8-NEXT: s_bfe_u32 s7, s2, 0x10003
; GFX8-NEXT: s_bfe_u32 s8, s2, 0x10001
@@ -7128,51 +7125,53 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX8-NEXT: s_bfe_u32 s10, s2, 0x10002
; GFX8-NEXT: s_bfe_u32 s11, s2, 0x10004
; GFX8-NEXT: s_bfe_u32 s12, s2, 0x10006
-; GFX8-NEXT: s_bfe_u32 s13, s2, 0x1000c
-; GFX8-NEXT: s_bfe_u32 s2, s2, 0x1000a
-; GFX8-NEXT: v_bfe_u32 v4, v2, 11, 1
-; GFX8-NEXT: v_lshrrev_b32_e32 v8, 15, v2
-; GFX8-NEXT: v_bfe_u32 v12, v2, 5, 1
-; GFX8-NEXT: v_bfe_u32 v6, v2, 14, 1
+; GFX8-NEXT: s_bfe_u32 s13, s2, 0x1000a
+; GFX8-NEXT: s_bfe_u32 s14, s2, 0x1000c
+; GFX8-NEXT: s_add_u32 s2, s0, 0x70
+; GFX8-NEXT: s_addc_u32 s3, s1, 0
+; GFX8-NEXT: v_and_b32_e32 v2, 0xffff, v0
+; GFX8-NEXT: v_mov_b32_e32 v7, s3
+; GFX8-NEXT: v_mov_b32_e32 v6, s2
+; GFX8-NEXT: s_add_u32 s2, s0, 0x60
+; GFX8-NEXT: v_lshrrev_b32_e32 v4, 15, v2
+; GFX8-NEXT: v_bfe_u32 v8, v2, 11, 1
+; GFX8-NEXT: v_bfe_u32 v9, v2, 5, 1
; GFX8-NEXT: v_bfe_u32 v0, v2, 8, 1
-; GFX8-NEXT: v_mov_b32_e32 v2, s2
+; GFX8-NEXT: v_bfe_u32 v2, v2, 14, 1
+; GFX8-NEXT: s_addc_u32 s3, s1, 0
+; GFX8-NEXT: flat_store_dwordx4 v[6:7], v[2:5]
+; GFX8-NEXT: v_mov_b32_e32 v7, s3
+; GFX8-NEXT: v_mov_b32_e32 v6, s2
; GFX8-NEXT: s_add_u32 s2, s0, 0x50
+; GFX8-NEXT: v_mov_b32_e32 v2, s14
+; GFX8-NEXT: v_mov_b32_e32 v4, s4
; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: v_mov_b32_e32 v11, s3
-; GFX8-NEXT: v_mov_b32_e32 v10, s2
+; GFX8-NEXT: flat_store_dwordx4 v[6:7], v[2:5]
+; GFX8-NEXT: v_mov_b32_e32 v7, s3
+; GFX8-NEXT: v_mov_b32_e32 v6, s2
; GFX8-NEXT: s_add_u32 s2, s0, 64
+; GFX8-NEXT: v_mov_b32_e32 v2, s13
+; GFX8-NEXT: v_mov_b32_e32 v4, v8
; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[2:5]
-; GFX8-NEXT: v_mov_b32_e32 v11, s3
-; GFX8-NEXT: v_mov_b32_e32 v10, s2
-; GFX8-NEXT: s_add_u32 s2, s0, 0x70
-; GFX8-NEXT: v_mov_b32_e32 v2, s4
-; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: flat_store_dwordx4 v[10:11], v[0:3]
-; GFX8-NEXT: v_mov_b32_e32 v4, s5
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
-; GFX8-NEXT: s_add_u32 s2, s0, 0x60
-; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[6:9]
-; GFX8-NEXT: v_mov_b32_e32 v0, s2
-; GFX8-NEXT: v_mov_b32_e32 v1, s3
+; GFX8-NEXT: flat_store_dwordx4 v[6:7], v[2:5]
+; GFX8-NEXT: v_mov_b32_e32 v7, s3
+; GFX8-NEXT: v_mov_b32_e32 v6, s2
; GFX8-NEXT: s_add_u32 s2, s0, 48
-; GFX8-NEXT: v_mov_b32_e32 v2, s13
+; GFX8-NEXT: v_mov_b32_e32 v2, s5
; GFX8-NEXT: s_addc_u32 s3, s1, 0
-; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
+; GFX8-NEXT: flat_store_dwordx4 v[6:7], v[0:3]
+; GFX8-NEXT: v_mov_b32_e32 v4, s6
; GFX8-NEXT: v_mov_b32_e32 v0, s2
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: s_add_u32 s2, s0, 32
; GFX8-NEXT: v_mov_b32_e32 v2, s12
-; GFX8-NEXT: v_mov_b32_e32 v4, s6
; GFX8-NEXT: s_addc_u32 s3, s1, 0
; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; GFX8-NEXT: v_mov_b32_e32 v0, s2
; GFX8-NEXT: v_mov_b32_e32 v1, s3
; GFX8-NEXT: s_add_u32 s2, s0, 16
; GFX8-NEXT: v_mov_b32_e32 v2, s11
-; GFX8-NEXT: v_mov_b32_e32 v4, v12
+; GFX8-NEXT: v_mov_b32_e32 v4, v9
; GFX8-NEXT: s_addc_u32 s3, s1, 0
; GFX8-NEXT: flat_store_dwordx4 v[0:1], v[2:5]
; GFX8-NEXT: v_mov_b32_e32 v0, s2
@@ -7266,32 +7265,32 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_u16 s2, s[2:3], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000a
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000e
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s3
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000b
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10009
+; GFX12-NEXT: s_lshr_b32 s4, s2, 15
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000d
; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, v1
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10008
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000c
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:112
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v0, s4
; GFX12-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-NEXT: s_lshr_b32 s3, s2, 15
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000e
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000b
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000a
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v0, s4
; GFX12-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000d
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000c
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:112
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10009
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10008
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v0, s4
; GFX12-NEXT: v_mov_b32_e32 v2, s3
; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10007
; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10006
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v0, s4
; GFX12-NEXT: v_mov_b32_e32 v2, s3
@@ -7326,29 +7325,29 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_u16 s2, s[2:3], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
-; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x1000a
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x1000e
; GFX1250-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s3
-; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000b
-; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10009
+; GFX1250-NEXT: s_lshr_b32 s4, s2, 15
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x1000d
; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, v1
-; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10008
-; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000c
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:112
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
-; GFX1250-NEXT: s_lshr_b32 s3, s2, 15
-; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000e
-; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x1000b
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000a
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
-; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x1000d
-; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x1000c
-; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:112
+; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10009
+; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10008
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10007
; GFX1250-NEXT: s_bfe_u32 s4, s2, 0x10006
-; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
+; GFX1250-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
; GFX1250-NEXT: s_wait_xcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v2, s3
; GFX1250-NEXT: s_bfe_u32 s3, s2, 0x10005
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
index 9189fdbadd2c94..5e038563d8f2b5 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i16.ll
@@ -6620,45 +6620,45 @@ define amdgpu_kernel void @constant_zextload_v16i16_to_v16i64(ptr addrspace(1) %
; GCN-NOHSA-SI-NEXT: s_mov_b32 s3, 0xf000
; GCN-NOHSA-SI-NEXT: s_mov_b32 s2, -1
; GCN-NOHSA-SI-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NOHSA-SI-NEXT: s_lshr_b32 s15, s9, 16
-; GCN-NOHSA-SI-NEXT: s_and_b32 s9, s9, 0xffff
-; GCN-NOHSA-SI-NEXT: s_lshr_b32 s14, s11, 16
+; GCN-NOHSA-SI-NEXT: s_lshr_b32 s15, s11, 16
; GCN-NOHSA-SI-NEXT: s_and_b32 s11, s11, 0xffff
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s9
+; GCN-NOHSA-SI-NEXT: s_lshr_b32 s14, s9, 16
+; GCN-NOHSA-SI-NEXT: s_and_b32 s9, s9, 0xffff
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s11
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s15
; GCN-NOHSA-SI-NEXT: s_lshr_b32 s13, s7, 16
; GCN-NOHSA-SI-NEXT: s_and_b32 s7, s7, 0xffff
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:80
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112
; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s11
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s9
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s14
; GCN-NOHSA-SI-NEXT: s_lshr_b32 s12, s5, 16
; GCN-NOHSA-SI-NEXT: s_and_b32 s5, s5, 0xffff
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:80
; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(0)
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s7
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s13
-; GCN-NOHSA-SI-NEXT: s_lshr_b32 s16, s8, 16
-; GCN-NOHSA-SI-NEXT: s_and_b32 s8, s8, 0xffff
+; GCN-NOHSA-SI-NEXT: s_lshr_b32 s16, s10, 16
+; GCN-NOHSA-SI-NEXT: s_and_b32 s10, s10, 0xffff
; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(0)
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s5
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s12
-; GCN-NOHSA-SI-NEXT: s_lshr_b32 s17, s10, 16
-; GCN-NOHSA-SI-NEXT: s_and_b32 s10, s10, 0xffff
+; GCN-NOHSA-SI-NEXT: s_lshr_b32 s17, s8, 16
+; GCN-NOHSA-SI-NEXT: s_and_b32 s8, s8, 0xffff
; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s8
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s10
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s16
; GCN-NOHSA-SI-NEXT: s_lshr_b32 s18, s6, 16
; GCN-NOHSA-SI-NEXT: s_and_b32 s6, s6, 0xffff
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:96
; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(0)
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s8
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s17
; GCN-NOHSA-SI-NEXT: s_lshr_b32 s19, s4, 16
; GCN-NOHSA-SI-NEXT: s_and_b32 s4, s4, 0xffff
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
; GCN-NOHSA-SI-NEXT: s_waitcnt expcnt(0)
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v0, s6
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v2, s18
@@ -6682,33 +6682,33 @@ define amdgpu_kernel void @constant_zextload_v16i16_to_v16i64(ptr addrspace(1) %
; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0)
; GCN-HSA-NEXT: s_lshr_b32 s12, s5, 16
; GCN-HSA-NEXT: s_lshr_b32 s13, s7, 16
-; GCN-HSA-NEXT: s_lshr_b32 s14, s11, 16
-; GCN-HSA-NEXT: s_lshr_b32 s2, s9, 16
-; GCN-HSA-NEXT: s_lshr_b32 s15, s8, 16
-; GCN-HSA-NEXT: s_lshr_b32 s16, s10, 16
+; GCN-HSA-NEXT: s_lshr_b32 s14, s9, 16
+; GCN-HSA-NEXT: s_lshr_b32 s2, s11, 16
+; GCN-HSA-NEXT: s_lshr_b32 s15, s10, 16
+; GCN-HSA-NEXT: s_lshr_b32 s16, s8, 16
; GCN-HSA-NEXT: s_lshr_b32 s17, s6, 16
; GCN-HSA-NEXT: s_lshr_b32 s18, s4, 16
; GCN-HSA-NEXT: s_and_b32 s4, s4, 0xffff
; GCN-HSA-NEXT: s_and_b32 s6, s6, 0xffff
-; GCN-HSA-NEXT: s_and_b32 s10, s10, 0xffff
; GCN-HSA-NEXT: s_and_b32 s8, s8, 0xffff
+; GCN-HSA-NEXT: s_and_b32 s10, s10, 0xffff
; GCN-HSA-NEXT: s_and_b32 s5, s5, 0xffff
; GCN-HSA-NEXT: s_and_b32 s7, s7, 0xffff
-; GCN-HSA-NEXT: s_and_b32 s11, s11, 0xffff
-; GCN-HSA-NEXT: s_and_b32 s3, s9, 0xffff
+; GCN-HSA-NEXT: s_and_b32 s9, s9, 0xffff
+; GCN-HSA-NEXT: s_and_b32 s3, s11, 0xffff
; GCN-HSA-NEXT: v_mov_b32_e32 v2, s2
-; GCN-HSA-NEXT: s_add_u32 s2, s0, 0x50
+; GCN-HSA-NEXT: s_add_u32 s2, s0, 0x70
; GCN-HSA-NEXT: v_mov_b32_e32 v0, s3
; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCN-HSA-NEXT: v_mov_b32_e32 v5, s3
; GCN-HSA-NEXT: v_mov_b32_e32 v4, s2
-; GCN-HSA-NEXT: s_add_u32 s2, s0, 0x70
+; GCN-HSA-NEXT: s_add_u32 s2, s0, 0x50
; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-HSA-NEXT: v_mov_b32_e32 v5, s3
; GCN-HSA-NEXT: v_mov_b32_e32 v4, s2
; GCN-HSA-NEXT: s_add_u32 s2, s0, 48
-; GCN-HSA-NEXT: v_mov_b32_e32 v0, s11
+; GCN-HSA-NEXT: v_mov_b32_e32 v0, s9
; GCN-HSA-NEXT: v_mov_b32_e32 v2, s14
; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
@@ -6721,22 +6721,22 @@ define amdgpu_kernel void @constant_zextload_v16i16_to_v16i64(ptr addrspace(1) %
; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-HSA-NEXT: v_mov_b32_e32 v5, s3
; GCN-HSA-NEXT: v_mov_b32_e32 v4, s2
-; GCN-HSA-NEXT: s_add_u32 s2, s0, 64
+; GCN-HSA-NEXT: s_add_u32 s2, s0, 0x60
; GCN-HSA-NEXT: v_mov_b32_e32 v0, s5
; GCN-HSA-NEXT: v_mov_b32_e32 v2, s12
; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-HSA-NEXT: v_mov_b32_e32 v5, s3
; GCN-HSA-NEXT: v_mov_b32_e32 v4, s2
-; GCN-HSA-NEXT: s_add_u32 s2, s0, 0x60
-; GCN-HSA-NEXT: v_mov_b32_e32 v0, s8
+; GCN-HSA-NEXT: s_add_u32 s2, s0, 64
+; GCN-HSA-NEXT: v_mov_b32_e32 v0, s10
; GCN-HSA-NEXT: v_mov_b32_e32 v2, s15
; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-HSA-NEXT: v_mov_b32_e32 v5, s3
; GCN-HSA-NEXT: v_mov_b32_e32 v4, s2
; GCN-HSA-NEXT: s_add_u32 s2, s0, 32
-; GCN-HSA-NEXT: v_mov_b32_e32 v0, s10
+; GCN-HSA-NEXT: v_mov_b32_e32 v0, s8
; GCN-HSA-NEXT: v_mov_b32_e32 v2, s16
; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
@@ -6768,41 +6768,41 @@ define amdgpu_kernel void @constant_zextload_v16i16_to_v16i64(ptr addrspace(1) %
; GCN-NOHSA-VI-NEXT: s_and_b32 s6, s6, 0xffff
; GCN-NOHSA-VI-NEXT: s_lshr_b32 s15, s7, 16
; GCN-NOHSA-VI-NEXT: s_and_b32 s7, s7, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s16, s10, 16
-; GCN-NOHSA-VI-NEXT: s_and_b32 s10, s10, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s17, s11, 16
-; GCN-NOHSA-VI-NEXT: s_and_b32 s11, s11, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s18, s8, 16
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s16, s8, 16
; GCN-NOHSA-VI-NEXT: s_and_b32 s8, s8, 0xffff
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s2, s9, 16
-; GCN-NOHSA-VI-NEXT: s_and_b32 s3, s9, 0xffff
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s17, s9, 16
+; GCN-NOHSA-VI-NEXT: s_and_b32 s9, s9, 0xffff
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s18, s10, 16
+; GCN-NOHSA-VI-NEXT: s_and_b32 s10, s10, 0xffff
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s2, s11, 16
+; GCN-NOHSA-VI-NEXT: s_and_b32 s3, s11, 0xffff
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s2
-; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s0, 0x50
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s0, 0x70
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s3
; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s1, 0
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
-; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s0, 64
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s0, 0x60
; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s1, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
-; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s0, 0x70
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s8
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s0, 0x50
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s10
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s18
; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s1, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
-; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s0, 0x60
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s11
+; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s0, 64
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s9
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s17
; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s1, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s3
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s2
; GCN-NOHSA-VI-NEXT: s_add_u32 s2, s0, 48
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s10
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s8
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s16
; GCN-NOHSA-VI-NEXT: s_addc_u32 s3, s1, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
@@ -6912,33 +6912,33 @@ define amdgpu_kernel void @constant_zextload_v16i16_to_v16i64(ptr addrspace(1) %
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: s_load_b256 s[0:7], s[10:11], 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_and_b32 s10, s5, 0xffff
+; GFX12-NEXT: s_and_b32 s10, s7, 0xffff
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v0, s10
-; GFX12-NEXT: s_lshr_b32 s5, s5, 16
+; GFX12-NEXT: s_lshr_b32 s7, s7, 16
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v3, v1
+; GFX12-NEXT: v_dual_mov_b32 v2, s7 :: v_dual_mov_b32 v3, v1
+; GFX12-NEXT: s_lshr_b32 s7, s6, 16
+; GFX12-NEXT: s_and_b32 s6, s6, 0xffff
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[8:9] offset:112
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s6
+; GFX12-NEXT: v_mov_b32_e32 v2, s7
+; GFX12-NEXT: s_lshr_b32 s6, s5, 16
+; GFX12-NEXT: s_and_b32 s5, s5, 0xffff
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[8:9] offset:96
+; GFX12-NEXT: v_mov_b32_e32 v0, s5
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v2, s6
; GFX12-NEXT: s_lshr_b32 s5, s4, 16
; GFX12-NEXT: s_and_b32 s4, s4, 0xffff
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[8:9] offset:80
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v0, s4
; GFX12-NEXT: v_mov_b32_e32 v2, s5
-; GFX12-NEXT: s_lshr_b32 s4, s7, 16
-; GFX12-NEXT: s_and_b32 s5, s7, 0xffff
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[8:9] offset:64
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s5
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
-; GFX12-NEXT: s_lshr_b32 s4, s6, 16
-; GFX12-NEXT: s_and_b32 s5, s6, 0xffff
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[8:9] offset:112
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s5
-; GFX12-NEXT: v_mov_b32_e32 v2, s4
; GFX12-NEXT: s_lshr_b32 s4, s3, 16
; GFX12-NEXT: s_and_b32 s3, s3, 0xffff
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[8:9] offset:96
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[8:9] offset:64
; GFX12-NEXT: v_mov_b32_e32 v0, s3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v2, s4
@@ -7175,57 +7175,58 @@ define amdgpu_kernel void @constant_sextload_v16i16_to_v16i64(ptr addrspace(1) %
; GCN-NOHSA-VI-NEXT: s_lshr_b32 s18, s1, 16
; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[16:17], s[2:3], 0x100000
; GCN-NOHSA-VI-NEXT: s_lshr_b32 s2, s2, 16
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[22:23], s[6:7], 0x100000
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s6, s6, 16
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[28:29], s[4:5], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[22:23], s[4:5], 0x100000
; GCN-NOHSA-VI-NEXT: s_lshr_b32 s4, s4, 16
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[28:29], s[6:7], 0x100000
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s6, s6, 16
; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[14:15], s[12:13], 0x100000
; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[12:13], s[18:19], 0x100000
; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[18:19], s[2:3], 0x100000
; GCN-NOHSA-VI-NEXT: s_mov_b32 s2, s3
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[24:25], s[6:7], 0x100000
-; GCN-NOHSA-VI-NEXT: s_mov_b32 s6, s7
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[30:31], s[4:5], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[24:25], s[4:5], 0x100000
; GCN-NOHSA-VI-NEXT: s_mov_b32 s4, s5
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[30:31], s[6:7], 0x100000
+; GCN-NOHSA-VI-NEXT: s_mov_b32 s6, s7
; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[10:11], s[0:1], 0x100000
; GCN-NOHSA-VI-NEXT: s_lshr_b32 s0, s0, 16
; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[20:21], s[2:3], 0x100000
; GCN-NOHSA-VI-NEXT: s_lshr_b32 s2, s3, 16
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[26:27], s[6:7], 0x100000
-; GCN-NOHSA-VI-NEXT: s_lshr_b32 s6, s7, 16
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[34:35], s[4:5], 0x100000
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[26:27], s[4:5], 0x100000
; GCN-NOHSA-VI-NEXT: s_lshr_b32 s4, s5, 16
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[34:35], s[6:7], 0x100000
+; GCN-NOHSA-VI-NEXT: s_lshr_b32 s6, s7, 16
; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[0:1], s[0:1], 0x100000
; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x100000
-; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x100000
; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x100000
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s4
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s8, 0x50
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s5
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s9, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s8, 64
+; GCN-NOHSA-VI-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x100000
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s6
+; GCN-NOHSA-VI-NEXT: s_add_u32 s6, s8, 0x70
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s7
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s7, s9, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s6
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s7
+; GCN-NOHSA-VI-NEXT: s_add_u32 s6, s8, 0x60
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s34
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s35
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s9, 0
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s7, s9, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s8, 0x70
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s6
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s28
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s29
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s30
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s31
-; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s9, 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s7
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
+; GCN-NOHSA-VI-NEXT: s_nop 0
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s4
+; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s8, 0x50
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s5
+; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s9, 0
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, s5
-; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s8, 0x60
+; GCN-NOHSA-VI-NEXT: s_add_u32 s4, s8, 64
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v0, s26
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, s27
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v2, s6
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, s7
; GCN-NOHSA-VI-NEXT: s_addc_u32 s5, s9, 0
; GCN-NOHSA-VI-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v4, s4
@@ -7359,16 +7360,16 @@ define amdgpu_kernel void @constant_sextload_v16i16_to_v16i64(ptr addrspace(1) %
; GFX12-NEXT: s_lshr_b32 s16, s1, 16
; GFX12-NEXT: s_mov_b32 s20, s3
; GFX12-NEXT: s_lshr_b32 s22, s3, 16
-; GFX12-NEXT: s_mov_b32 s28, s7
-; GFX12-NEXT: s_lshr_b32 s30, s7, 16
-; GFX12-NEXT: s_mov_b32 s38, s5
-; GFX12-NEXT: s_lshr_b32 s40, s5, 16
+; GFX12-NEXT: s_mov_b32 s28, s5
+; GFX12-NEXT: s_lshr_b32 s30, s5, 16
+; GFX12-NEXT: s_mov_b32 s38, s7
+; GFX12-NEXT: s_lshr_b32 s40, s7, 16
; GFX12-NEXT: s_lshr_b32 s12, s0, 16
; GFX12-NEXT: s_lshr_b32 s18, s2, 16
-; GFX12-NEXT: s_lshr_b32 s26, s6, 16
-; GFX12-NEXT: s_lshr_b32 s36, s4, 16
-; GFX12-NEXT: s_bfe_i64 s[24:25], s[6:7], 0x100000
-; GFX12-NEXT: s_bfe_i64 s[34:35], s[4:5], 0x100000
+; GFX12-NEXT: s_lshr_b32 s26, s4, 16
+; GFX12-NEXT: s_lshr_b32 s36, s6, 16
+; GFX12-NEXT: s_bfe_i64 s[24:25], s[4:5], 0x100000
+; GFX12-NEXT: s_bfe_i64 s[34:35], s[6:7], 0x100000
; GFX12-NEXT: s_bfe_i64 s[6:7], s[14:15], 0x100000
; GFX12-NEXT: s_bfe_i64 s[4:5], s[16:17], 0x100000
; GFX12-NEXT: s_bfe_i64 s[14:15], s[20:21], 0x100000
@@ -7395,8 +7396,8 @@ define amdgpu_kernel void @constant_sextload_v16i16_to_v16i64(ptr addrspace(1) %
; GFX12-NEXT: v_dual_mov_b32 v16, s14 :: v_dual_mov_b32 v19, s17
; GFX12-NEXT: v_mov_b32_e32 v18, s16
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: global_store_b128 v24, v[8:11], s[8:9] offset:80
-; GFX12-NEXT: global_store_b128 v24, v[0:3], s[8:9] offset:64
+; GFX12-NEXT: global_store_b128 v24, v[8:11], s[8:9] offset:112
+; GFX12-NEXT: global_store_b128 v24, v[0:3], s[8:9] offset:96
; GFX12-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s13
; GFX12-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s12
; GFX12-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
@@ -7404,8 +7405,8 @@ define amdgpu_kernel void @constant_sextload_v16i16_to_v16i64(ptr addrspace(1) %
; GFX12-NEXT: v_dual_mov_b32 v21, s11 :: v_dual_mov_b32 v20, s10
; GFX12-NEXT: v_dual_mov_b32 v23, s3 :: v_dual_mov_b32 v22, s2
; GFX12-NEXT: s_clause 0x5
-; GFX12-NEXT: global_store_b128 v24, v[12:15], s[8:9] offset:112
-; GFX12-NEXT: global_store_b128 v24, v[4:7], s[8:9] offset:96
+; GFX12-NEXT: global_store_b128 v24, v[12:15], s[8:9] offset:80
+; GFX12-NEXT: global_store_b128 v24, v[4:7], s[8:9] offset:64
; GFX12-NEXT: global_store_b128 v24, v[16:19], s[8:9] offset:48
; GFX12-NEXT: global_store_b128 v24, v[0:3], s[8:9] offset:32
; GFX12-NEXT: global_store_b128 v24, v[8:11], s[8:9] offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
index fba7b61f960cc4..20e94555e67882 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
@@ -6528,53 +6528,53 @@ define amdgpu_kernel void @constant_zextload_v16i8_to_v16i64(ptr addrspace(1) %o
; GFX6-NOHSA-NEXT: s_mov_b32 s3, 0xf000
; GFX6-NOHSA-NEXT: s_mov_b32 s2, -1
; GFX6-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NOHSA-NEXT: s_lshr_b32 s11, s6, 24
-; GFX6-NOHSA-NEXT: s_bfe_u32 s12, s6, 0x80008
-; GFX6-NOHSA-NEXT: s_and_b32 s19, s6, 0xff
-; GFX6-NOHSA-NEXT: s_bfe_u32 s6, s6, 0x80010
-; GFX6-NOHSA-NEXT: s_lshr_b32 s10, s7, 24
-; GFX6-NOHSA-NEXT: s_bfe_u32 s13, s7, 0x80008
-; GFX6-NOHSA-NEXT: s_and_b32 s18, s7, 0xff
+; GFX6-NOHSA-NEXT: s_lshr_b32 s11, s7, 24
+; GFX6-NOHSA-NEXT: s_bfe_u32 s12, s7, 0x80008
+; GFX6-NOHSA-NEXT: s_and_b32 s19, s7, 0xff
; GFX6-NOHSA-NEXT: s_bfe_u32 s7, s7, 0x80010
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s6
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s11
-; GFX6-NOHSA-NEXT: s_lshr_b32 s9, s4, 24
-; GFX6-NOHSA-NEXT: s_bfe_u32 s14, s4, 0x80008
-; GFX6-NOHSA-NEXT: s_and_b32 s17, s4, 0xff
-; GFX6-NOHSA-NEXT: s_bfe_u32 s4, s4, 0x80010
-; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:80
-; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0)
+; GFX6-NOHSA-NEXT: s_lshr_b32 s10, s6, 24
+; GFX6-NOHSA-NEXT: s_bfe_u32 s13, s6, 0x80008
+; GFX6-NOHSA-NEXT: s_and_b32 s18, s6, 0xff
+; GFX6-NOHSA-NEXT: s_bfe_u32 s6, s6, 0x80010
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s7
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s10
-; GFX6-NOHSA-NEXT: s_lshr_b32 s8, s5, 24
-; GFX6-NOHSA-NEXT: s_bfe_u32 s15, s5, 0x80008
-; GFX6-NOHSA-NEXT: s_and_b32 s16, s5, 0xff
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s11
+; GFX6-NOHSA-NEXT: s_lshr_b32 s9, s5, 24
+; GFX6-NOHSA-NEXT: s_bfe_u32 s14, s5, 0x80008
+; GFX6-NOHSA-NEXT: s_and_b32 s17, s5, 0xff
; GFX6-NOHSA-NEXT: s_bfe_u32 s5, s5, 0x80010
; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112
; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0)
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s4
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s9
-; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s6
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s10
+; GFX6-NOHSA-NEXT: s_lshr_b32 s8, s4, 24
+; GFX6-NOHSA-NEXT: s_bfe_u32 s15, s4, 0x80008
+; GFX6-NOHSA-NEXT: s_and_b32 s16, s4, 0xff
+; GFX6-NOHSA-NEXT: s_bfe_u32 s4, s4, 0x80010
+; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:80
; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0)
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s5
-; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s9
; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0)
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s4
+; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s8
+; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:16
+; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0)
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s19
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s12
-; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
+; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:96
; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0)
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s18
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s13
-; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:96
+; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0)
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s17
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s14
-; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
+; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
; GFX6-NOHSA-NEXT: s_waitcnt expcnt(0)
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v0, s16
; GFX6-NOHSA-NEXT: v_mov_b32_e32 v2, s15
-; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:32
+; GFX6-NOHSA-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0
; GFX6-NOHSA-NEXT: s_endpgm
;
; GFX7-HSA-LABEL: constant_zextload_v16i8_to_v16i64:
@@ -6588,70 +6588,70 @@ define amdgpu_kernel void @constant_zextload_v16i8_to_v16i64(ptr addrspace(1) %o
; GFX7-HSA-NEXT: s_load_dwordx4 s[4:7], s[2:3], 0x0
; GFX7-HSA-NEXT: v_mov_b32_e32 v3, v1
; GFX7-HSA-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-HSA-NEXT: s_lshr_b32 s8, s5, 24
-; GFX7-HSA-NEXT: s_lshr_b32 s9, s4, 24
-; GFX7-HSA-NEXT: s_lshr_b32 s10, s7, 24
-; GFX7-HSA-NEXT: s_lshr_b32 s2, s6, 24
-; GFX7-HSA-NEXT: s_bfe_u32 s11, s6, 0x80008
-; GFX7-HSA-NEXT: s_bfe_u32 s12, s7, 0x80008
-; GFX7-HSA-NEXT: s_bfe_u32 s13, s4, 0x80008
-; GFX7-HSA-NEXT: s_bfe_u32 s14, s5, 0x80008
-; GFX7-HSA-NEXT: s_and_b32 s15, s5, 0xff
-; GFX7-HSA-NEXT: s_and_b32 s16, s4, 0xff
-; GFX7-HSA-NEXT: s_and_b32 s17, s7, 0xff
-; GFX7-HSA-NEXT: s_and_b32 s18, s6, 0xff
-; GFX7-HSA-NEXT: s_bfe_u32 s5, s5, 0x80010
+; GFX7-HSA-NEXT: s_lshr_b32 s8, s4, 24
+; GFX7-HSA-NEXT: s_lshr_b32 s9, s5, 24
+; GFX7-HSA-NEXT: s_lshr_b32 s10, s6, 24
+; GFX7-HSA-NEXT: s_lshr_b32 s2, s7, 24
+; GFX7-HSA-NEXT: s_bfe_u32 s11, s7, 0x80008
+; GFX7-HSA-NEXT: s_bfe_u32 s12, s6, 0x80008
+; GFX7-HSA-NEXT: s_bfe_u32 s13, s5, 0x80008
+; GFX7-HSA-NEXT: s_bfe_u32 s14, s4, 0x80008
+; GFX7-HSA-NEXT: s_and_b32 s15, s4, 0xff
+; GFX7-HSA-NEXT: s_and_b32 s16, s5, 0xff
+; GFX7-HSA-NEXT: s_and_b32 s17, s6, 0xff
+; GFX7-HSA-NEXT: s_and_b32 s18, s7, 0xff
; GFX7-HSA-NEXT: s_bfe_u32 s4, s4, 0x80010
-; GFX7-HSA-NEXT: s_bfe_u32 s7, s7, 0x80010
-; GFX7-HSA-NEXT: s_bfe_u32 s3, s6, 0x80010
+; GFX7-HSA-NEXT: s_bfe_u32 s5, s5, 0x80010
+; GFX7-HSA-NEXT: s_bfe_u32 s6, s6, 0x80010
+; GFX7-HSA-NEXT: s_bfe_u32 s3, s7, 0x80010
; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s2
-; GFX7-HSA-NEXT: s_add_u32 s2, s0, 0x50
+; GFX7-HSA-NEXT: s_add_u32 s2, s0, 0x70
; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s3
; GFX7-HSA-NEXT: s_addc_u32 s3, s1, 0
; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s3
; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s2
-; GFX7-HSA-NEXT: s_add_u32 s2, s0, 0x70
+; GFX7-HSA-NEXT: s_add_u32 s2, s0, 0x50
; GFX7-HSA-NEXT: s_addc_u32 s3, s1, 0
; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s3
; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s2
-; GFX7-HSA-NEXT: s_add_u32 s2, s0, 16
-; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s7
+; GFX7-HSA-NEXT: s_add_u32 s2, s0, 48
+; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s6
; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s10
; GFX7-HSA-NEXT: s_addc_u32 s3, s1, 0
; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s3
; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s2
-; GFX7-HSA-NEXT: s_add_u32 s2, s0, 48
-; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s4
+; GFX7-HSA-NEXT: s_add_u32 s2, s0, 16
+; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s5
; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s9
; GFX7-HSA-NEXT: s_addc_u32 s3, s1, 0
; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s3
; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s2
-; GFX7-HSA-NEXT: s_add_u32 s2, s0, 64
-; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s5
+; GFX7-HSA-NEXT: s_add_u32 s2, s0, 0x60
+; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s4
; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s8
; GFX7-HSA-NEXT: s_addc_u32 s3, s1, 0
; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s3
; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s2
-; GFX7-HSA-NEXT: s_add_u32 s2, s0, 0x60
+; GFX7-HSA-NEXT: s_add_u32 s2, s0, 64
; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s18
; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s11
; GFX7-HSA-NEXT: s_addc_u32 s3, s1, 0
; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s3
+; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s2
+; GFX7-HSA-NEXT: s_add_u32 s2, s0, 32
; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s17
; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s12
-; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s2
+; GFX7-HSA-NEXT: s_addc_u32 s3, s1, 0
; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
-; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s1
-; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s0
-; GFX7-HSA-NEXT: s_add_u32 s0, s0, 32
+; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s3
; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s16
; GFX7-HSA-NEXT: v_mov_b32_e32 v2, s13
-; GFX7-HSA-NEXT: s_addc_u32 s1, s1, 0
+; GFX7-HSA-NEXT: v_mov_b32_e32 v4, s2
; GFX7-HSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX7-HSA-NEXT: v_mov_b32_e32 v5, s1
; GFX7-HSA-NEXT: v_mov_b32_e32 v0, s15
@@ -6670,18 +6670,18 @@ define amdgpu_kernel void @constant_zextload_v16i8_to_v16i64(ptr addrspace(1) %o
; GFX8-NOHSA-NEXT: s_waitcnt lgkmcnt(0)
; GFX8-NOHSA-NEXT: s_lshr_b32 s8, s5, 24
; GFX8-NOHSA-NEXT: s_lshr_b32 s2, s7, 24
-; GFX8-NOHSA-NEXT: s_lshr_b32 s9, s6, 24
-; GFX8-NOHSA-NEXT: s_bfe_u32 s10, s6, 0x80008
-; GFX8-NOHSA-NEXT: s_bfe_u32 s11, s7, 0x80008
+; GFX8-NOHSA-NEXT: s_bfe_u32 s9, s7, 0x80008
+; GFX8-NOHSA-NEXT: s_lshr_b32 s10, s6, 24
+; GFX8-NOHSA-NEXT: s_bfe_u32 s11, s6, 0x80008
; GFX8-NOHSA-NEXT: s_bfe_u32 s12, s5, 0x80008
; GFX8-NOHSA-NEXT: s_lshr_b32 s13, s4, 24
; GFX8-NOHSA-NEXT: s_bfe_u32 s14, s4, 0x80008
; GFX8-NOHSA-NEXT: s_and_b32 s15, s4, 0xff
; GFX8-NOHSA-NEXT: s_bfe_u32 s4, s4, 0x80010
; GFX8-NOHSA-NEXT: s_and_b32 s16, s5, 0xff
-; GFX8-NOHSA-NEXT: s_and_b32 s17, s7, 0xff
-; GFX8-NOHSA-NEXT: s_and_b32 s18, s6, 0xff
+; GFX8-NOHSA-NEXT: s_and_b32 s17, s6, 0xff
; GFX8-NOHSA-NEXT: s_bfe_u32 s6, s6, 0x80010
+; GFX8-NOHSA-NEXT: s_and_b32 s18, s7, 0xff
; GFX8-NOHSA-NEXT: s_bfe_u32 s5, s5, 0x80010
; GFX8-NOHSA-NEXT: s_bfe_u32 s3, s7, 0x80010
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s2
@@ -6695,22 +6695,22 @@ define amdgpu_kernel void @constant_zextload_v16i8_to_v16i64(ptr addrspace(1) %o
; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s3
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NOHSA-NEXT: s_add_u32 s2, s0, 0x50
+; GFX8-NOHSA-NEXT: s_add_u32 s2, s0, 0x60
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s5
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s8
; GFX8-NOHSA-NEXT: s_addc_u32 s3, s1, 0
; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s3
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NOHSA-NEXT: s_add_u32 s2, s0, 64
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s6
+; GFX8-NOHSA-NEXT: s_add_u32 s2, s0, 0x50
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s18
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s9
; GFX8-NOHSA-NEXT: s_addc_u32 s3, s1, 0
; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v5, s3
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v4, s2
-; GFX8-NOHSA-NEXT: s_add_u32 s2, s0, 0x60
-; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s18
+; GFX8-NOHSA-NEXT: s_add_u32 s2, s0, 64
+; GFX8-NOHSA-NEXT: v_mov_b32_e32 v0, s6
; GFX8-NOHSA-NEXT: v_mov_b32_e32 v2, s10
; GFX8-NOHSA-NEXT: s_addc_u32 s3, s1, 0
; GFX8-NOHSA-NEXT: flat_store_dwordx4 v[4:5], v[0:3]
@@ -6837,9 +6837,15 @@ define amdgpu_kernel void @constant_zextload_v16i8_to_v16i64(ptr addrspace(1) %o
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v0, s3
; GFX12-NEXT: v_mov_b32_e32 v2, s2
+; GFX12-NEXT: s_bfe_u32 s2, s7, 0x80008
+; GFX12-NEXT: s_and_b32 s3, s7, 0xff
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s3
+; GFX12-NEXT: v_mov_b32_e32 v2, s2
; GFX12-NEXT: s_lshr_b32 s2, s6, 24
; GFX12-NEXT: s_bfe_u32 s3, s6, 0x80010
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:48
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v0, s3
; GFX12-NEXT: v_mov_b32_e32 v2, s2
@@ -6849,15 +6855,9 @@ define amdgpu_kernel void @constant_zextload_v16i8_to_v16i64(ptr addrspace(1) %o
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v0, s3
; GFX12-NEXT: v_mov_b32_e32 v2, s2
-; GFX12-NEXT: s_bfe_u32 s2, s7, 0x80008
-; GFX12-NEXT: s_and_b32 s3, s7, 0xff
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_mov_b32_e32 v0, s3
-; GFX12-NEXT: v_mov_b32_e32 v2, s2
; GFX12-NEXT: s_bfe_u32 s2, s5, 0x80008
; GFX12-NEXT: s_and_b32 s3, s5, 0xff
-; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:96
+; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:64
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v0, s3
; GFX12-NEXT: v_mov_b32_e32 v2, s2
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
index 9b43e1e70f532d..3dc72d59f99b0b 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i16.ll
@@ -6423,31 +6423,31 @@ define amdgpu_kernel void @global_zextload_v16i16_to_v16i64(ptr addrspace(1) %ou
; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(1)
; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v10, 16, v1
; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v30, 16, v5
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v28, 0xffff, v5
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v30, 16, v7
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v28, 0xffff, v7
; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v14, 16, v3
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v18, 16, v4
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v18, 16, v6
; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v22, 16, v2
; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v26, 16, v0
; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v24, 0xffff, v0
; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v20, 0xffff, v2
; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v8, 0xffff, v1
; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v12, 0xffff, v3
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v2, 16, v6
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v0, 0xffff, v6
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v16, 0xffff, v4
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v2, 16, v4
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v16, 0xffff, v6
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v4, 0xffff, v5
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v5, v29
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v7, v29
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v1, v29
; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v3, v29
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:80
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:80
; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:64
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:64
; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:32
; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0
; GCN-NOHSA-SI-NEXT: s_endpgm
@@ -6474,7 +6474,7 @@ define amdgpu_kernel void @global_zextload_v16i16_to_v16i64(ptr addrspace(1) %ou
; GCN-HSA-NEXT: s_addc_u32 s5, s1, 0
; GCN-HSA-NEXT: v_mov_b32_e32 v22, s5
; GCN-HSA-NEXT: v_mov_b32_e32 v21, s4
-; GCN-HSA-NEXT: s_add_u32 s4, s0, 0x50
+; GCN-HSA-NEXT: s_add_u32 s4, s0, 0x70
; GCN-HSA-NEXT: v_mov_b32_e32 v12, v8
; GCN-HSA-NEXT: v_mov_b32_e32 v14, v8
; GCN-HSA-NEXT: s_addc_u32 s5, s1, 0
@@ -6484,61 +6484,61 @@ define amdgpu_kernel void @global_zextload_v16i16_to_v16i64(ptr addrspace(1) %ou
; GCN-HSA-NEXT: v_mov_b32_e32 v20, v8
; GCN-HSA-NEXT: v_mov_b32_e32 v10, v8
; GCN-HSA-NEXT: v_mov_b32_e32 v23, v8
+; GCN-HSA-NEXT: v_mov_b32_e32 v25, v8
+; GCN-HSA-NEXT: v_mov_b32_e32 v27, v8
; GCN-HSA-NEXT: s_waitcnt vmcnt(1)
; GCN-HSA-NEXT: v_lshrrev_b32_e32 v13, 16, v1
; GCN-HSA-NEXT: v_and_b32_e32 v11, 0xffff, v1
; GCN-HSA-NEXT: flat_store_dwordx4 v[21:22], v[11:14]
; GCN-HSA-NEXT: v_mov_b32_e32 v22, s5
; GCN-HSA-NEXT: v_mov_b32_e32 v21, s4
-; GCN-HSA-NEXT: s_add_u32 s4, s0, 0x70
+; GCN-HSA-NEXT: s_add_u32 s4, s0, 0x50
; GCN-HSA-NEXT: s_waitcnt vmcnt(1)
-; GCN-HSA-NEXT: v_lshrrev_b32_e32 v16, 16, v5
-; GCN-HSA-NEXT: v_and_b32_e32 v14, 0xffff, v5
+; GCN-HSA-NEXT: v_lshrrev_b32_e32 v16, 16, v7
+; GCN-HSA-NEXT: v_and_b32_e32 v14, 0xffff, v7
; GCN-HSA-NEXT: s_addc_u32 s5, s1, 0
; GCN-HSA-NEXT: flat_store_dwordx4 v[21:22], v[14:17]
-; GCN-HSA-NEXT: v_lshrrev_b32_e32 v19, 16, v7
+; GCN-HSA-NEXT: v_lshrrev_b32_e32 v19, 16, v5
; GCN-HSA-NEXT: v_mov_b32_e32 v15, s5
; GCN-HSA-NEXT: v_mov_b32_e32 v14, s4
-; GCN-HSA-NEXT: v_and_b32_e32 v17, 0xffff, v7
+; GCN-HSA-NEXT: v_and_b32_e32 v17, 0xffff, v5
; GCN-HSA-NEXT: flat_store_dwordx4 v[14:15], v[17:20]
; GCN-HSA-NEXT: v_lshrrev_b32_e32 v9, 16, v3
; GCN-HSA-NEXT: v_mov_b32_e32 v19, s3
; GCN-HSA-NEXT: v_mov_b32_e32 v18, s2
; GCN-HSA-NEXT: s_add_u32 s2, s0, 32
-; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
-; GCN-HSA-NEXT: v_mov_b32_e32 v25, s3
-; GCN-HSA-NEXT: v_mov_b32_e32 v24, s2
-; GCN-HSA-NEXT: s_add_u32 s2, s0, 64
; GCN-HSA-NEXT: v_and_b32_e32 v7, 0xffff, v3
; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCN-HSA-NEXT: flat_store_dwordx4 v[18:19], v[7:10]
-; GCN-HSA-NEXT: v_mov_b32_e32 v19, s1
-; GCN-HSA-NEXT: v_mov_b32_e32 v18, s0
-; GCN-HSA-NEXT: s_add_u32 s0, s0, 0x60
+; GCN-HSA-NEXT: v_mov_b32_e32 v19, s3
+; GCN-HSA-NEXT: v_mov_b32_e32 v18, s2
+; GCN-HSA-NEXT: s_add_u32 s2, s0, 0x60
+; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
; GCN-HSA-NEXT: v_mov_b32_e32 v11, v8
; GCN-HSA-NEXT: v_mov_b32_e32 v13, v8
-; GCN-HSA-NEXT: s_addc_u32 s1, s1, 0
-; GCN-HSA-NEXT: v_lshrrev_b32_e32 v12, 16, v2
-; GCN-HSA-NEXT: v_and_b32_e32 v10, 0xffff, v2
; GCN-HSA-NEXT: v_mov_b32_e32 v21, v8
; GCN-HSA-NEXT: v_mov_b32_e32 v15, v8
; GCN-HSA-NEXT: v_mov_b32_e32 v17, v8
-; GCN-HSA-NEXT: v_mov_b32_e32 v7, v8
-; GCN-HSA-NEXT: v_mov_b32_e32 v9, v8
-; GCN-HSA-NEXT: v_lshrrev_b32_e32 v16, 16, v4
-; GCN-HSA-NEXT: v_lshrrev_b32_e32 v8, 16, v6
-; GCN-HSA-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GCN-HSA-NEXT: v_and_b32_e32 v14, 0xffff, v4
+; GCN-HSA-NEXT: v_mov_b32_e32 v8, s1
+; GCN-HSA-NEXT: v_mov_b32_e32 v7, s0
+; GCN-HSA-NEXT: s_add_u32 s0, s0, 64
+; GCN-HSA-NEXT: s_addc_u32 s1, s1, 0
+; GCN-HSA-NEXT: v_lshrrev_b32_e32 v12, 16, v2
+; GCN-HSA-NEXT: v_and_b32_e32 v10, 0xffff, v2
+; GCN-HSA-NEXT: v_lshrrev_b32_e32 v16, 16, v6
+; GCN-HSA-NEXT: v_lshrrev_b32_e32 v26, 16, v4
+; GCN-HSA-NEXT: v_and_b32_e32 v24, 0xffff, v4
+; GCN-HSA-NEXT: v_and_b32_e32 v14, 0xffff, v6
; GCN-HSA-NEXT: v_mov_b32_e32 v4, s3
; GCN-HSA-NEXT: v_mov_b32_e32 v3, s2
-; GCN-HSA-NEXT: v_mov_b32_e32 v27, s1
-; GCN-HSA-NEXT: v_mov_b32_e32 v26, s0
+; GCN-HSA-NEXT: v_mov_b32_e32 v6, s1
+; GCN-HSA-NEXT: v_mov_b32_e32 v5, s0
; GCN-HSA-NEXT: v_lshrrev_b32_e32 v22, 16, v0
; GCN-HSA-NEXT: v_and_b32_e32 v20, 0xffff, v0
-; GCN-HSA-NEXT: flat_store_dwordx4 v[24:25], v[10:13]
-; GCN-HSA-NEXT: flat_store_dwordx4 v[18:19], v[20:23]
+; GCN-HSA-NEXT: flat_store_dwordx4 v[18:19], v[10:13]
+; GCN-HSA-NEXT: flat_store_dwordx4 v[7:8], v[20:23]
; GCN-HSA-NEXT: flat_store_dwordx4 v[3:4], v[14:17]
-; GCN-HSA-NEXT: flat_store_dwordx4 v[26:27], v[6:9]
+; GCN-HSA-NEXT: flat_store_dwordx4 v[5:6], v[24:27]
; GCN-HSA-NEXT: s_endpgm
;
; GCN-NOHSA-VI-LABEL: global_zextload_v16i16_to_v16i64:
@@ -6553,25 +6553,21 @@ define amdgpu_kernel void @global_zextload_v16i16_to_v16i64(ptr addrspace(1) %ou
; GCN-NOHSA-VI-NEXT: s_mov_b32 s9, s7
; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0 offset:16
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v29, 0
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v31, v29
; GCN-NOHSA-VI-NEXT: s_mov_b32 s0, s4
; GCN-NOHSA-VI-NEXT: s_mov_b32 s1, s5
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v25, v29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v27, v29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v21, v29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v23, v29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v17, v29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v19, v29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v13, v29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v15, v29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v9, v29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v11, v29
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(1)
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v10, 16, v0
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v30, 16, v5
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v28, 0xffff, v5
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v26, 16, v5
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v24, 0xffff, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, 0
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v22, 16, v4
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v20, 0xffff, v4
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v30, 16, v6
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v28, 0xffff, v6
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v6, 16, v7
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v4, 0xffff, v7
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v7, v5
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v8, 0xffff, v0
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v14, 16, v1
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v12, 0xffff, v1
@@ -6579,20 +6575,24 @@ define amdgpu_kernel void @global_zextload_v16i16_to_v16i64(ptr addrspace(1) %ou
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v16, 0xffff, v2
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v22, 16, v6
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v20, 0xffff, v6
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v26, 16, v7
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v24, 0xffff, v7
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v5, v29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v7, v29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, v29
-; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, v29
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:80
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:64
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:112
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v29, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v31, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v25, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v27, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v21, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v23, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v17, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v19, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v13, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v15, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v9, v5
+; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v11, v5
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:112
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:64
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:32
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:16
@@ -6988,38 +6988,38 @@ define amdgpu_kernel void @global_sextload_v16i16_to_v16i64(ptr addrspace(1) %ou
; GCN-NOHSA-VI-NEXT: s_mov_b32 s8, s6
; GCN-NOHSA-VI-NEXT: s_mov_b32 s9, s7
; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[4:7], off, s[8:11], 0
-; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[13:16], off, s[8:11], 0 offset:16
+; GCN-NOHSA-VI-NEXT: buffer_load_dwordx4 v[14:17], off, s[8:11], 0 offset:16
; GCN-NOHSA-VI-NEXT: s_mov_b32 s0, s4
; GCN-NOHSA-VI-NEXT: s_mov_b32 s1, s5
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(1)
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GCN-NOHSA-VI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v14
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 16, v17
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v19, v2, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v17, v14, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v17, v17, 0, 16
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v18, 31, v17
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v20, 31, v19
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v2, v1, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v15
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v14
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:112
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v0, v4, 0, 16
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v19, v1, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v13
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v1, 16, v16
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v4, v5, 0, 16
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v8, v6, 0, 16
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v5, 16, v6
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v12, v7, 0, 16
; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v7, 16, v7
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v6, v3, 0, 16
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 16, v16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v25, v13, 0, 16
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v3, 16, v15
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v25, v16, 0, 16
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v27, v1, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v17, v15, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v17, v14, 0, 16
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v10, v5, 0, 16
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v14, v7, 0, 16
; GCN-NOHSA-VI-NEXT: v_bfe_i32 v23, v3, 0, 16
-; GCN-NOHSA-VI-NEXT: v_bfe_i32 v21, v16, 0, 16
+; GCN-NOHSA-VI-NEXT: v_bfe_i32 v21, v15, 0, 16
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v26, 31, v25
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v28, 31, v27
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v1, 31, v0
@@ -7034,9 +7034,9 @@ define amdgpu_kernel void @global_sextload_v16i16_to_v16i64(ptr addrspace(1) %ou
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v15, 31, v14
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v20, 31, v19
; GCN-NOHSA-VI-NEXT: v_ashrrev_i32_e32 v24, 31, v23
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[25:28], off, s[0:3], 0 offset:64
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[21:24], off, s[0:3], 0 offset:112
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[25:28], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[21:24], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[17:20], off, s[0:3], 0 offset:64
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:48
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/load-global-i8.ll b/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
index 626cd360f4ded0..e41ca502156f08 100644
--- a/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-global-i8.ll
@@ -6351,49 +6351,49 @@ define amdgpu_kernel void @global_zextload_v16i8_to_v16i64(ptr addrspace(1) %out
; GCN-NOHSA-SI-NEXT: s_mov_b32 s8, s6
; GCN-NOHSA-SI-NEXT: s_mov_b32 s9, s7
; GCN-NOHSA-SI-NEXT: buffer_load_dwordx4 v[0:3], off, s[8:11], 0
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v33, 0
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v35, v33
; GCN-NOHSA-SI-NEXT: s_mov_b32 s0, s4
; GCN-NOHSA-SI-NEXT: s_mov_b32 s1, s5
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v29, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v31, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v25, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v27, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v21, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v23, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v17, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v19, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v5, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v7, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v9, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v11, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v13, v33
-; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v15, v33
; GCN-NOHSA-SI-NEXT: s_waitcnt vmcnt(0)
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v34, 24, v2
-; GCN-NOHSA-SI-NEXT: v_bfe_u32 v32, v2, 16, 8
-; GCN-NOHSA-SI-NEXT: v_bfe_u32 v6, v3, 8, 8
-; GCN-NOHSA-SI-NEXT: v_bfe_u32 v10, v0, 8, 8
-; GCN-NOHSA-SI-NEXT: v_bfe_u32 v14, v1, 8, 8
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v12, 0xff, v1
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v8, 0xff, v0
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v4, 0xff, v3
-; GCN-NOHSA-SI-NEXT: v_bfe_u32 v18, v2, 8, 8
-; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v16, 0xff, v2
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v22, 24, v1
-; GCN-NOHSA-SI-NEXT: v_bfe_u32 v20, v1, 16, 8
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v26, 24, v0
-; GCN-NOHSA-SI-NEXT: v_bfe_u32 v24, v0, 16, 8
-; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v30, 24, v3
-; GCN-NOHSA-SI-NEXT: v_bfe_u32 v28, v3, 16, 8
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[32:35], off, s[0:3], 0 offset:80
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:112
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:16
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:48
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:64
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:96
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0
-; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT: v_bfe_u32 v10, v1, 8, 8
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v8, 0xff, v1
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v26, 24, v1
+; GCN-NOHSA-SI-NEXT: v_bfe_u32 v24, v1, 16, 8
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v1, 0
+; GCN-NOHSA-SI-NEXT: v_bfe_u32 v6, v2, 8, 8
+; GCN-NOHSA-SI-NEXT: v_bfe_u32 v14, v0, 8, 8
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v12, 0xff, v0
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v4, 0xff, v2
+; GCN-NOHSA-SI-NEXT: v_bfe_u32 v18, v3, 8, 8
+; GCN-NOHSA-SI-NEXT: v_and_b32_e32 v16, 0xff, v3
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v22, 24, v0
+; GCN-NOHSA-SI-NEXT: v_bfe_u32 v20, v0, 16, 8
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v30, 24, v2
+; GCN-NOHSA-SI-NEXT: v_bfe_u32 v28, v2, 16, 8
+; GCN-NOHSA-SI-NEXT: v_lshrrev_b32_e32 v2, 24, v3
+; GCN-NOHSA-SI-NEXT: v_bfe_u32 v0, v3, 16, 8
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v3, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v29, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v31, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v25, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v27, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v21, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v23, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v17, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v19, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v5, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v7, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v9, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v11, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v13, v1
+; GCN-NOHSA-SI-NEXT: v_mov_b32_e32 v15, v1
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:112
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:80
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:48
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:16
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:96
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:64
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:32
+; GCN-NOHSA-SI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0
; GCN-NOHSA-SI-NEXT: s_endpgm
;
; GCN-HSA-LABEL: global_zextload_v16i8_to_v16i64:
@@ -6402,81 +6402,81 @@ define amdgpu_kernel void @global_zextload_v16i8_to_v16i64(ptr addrspace(1) %out
; GCN-HSA-NEXT: s_add_i32 s12, s12, s17
; GCN-HSA-NEXT: s_mov_b32 flat_scratch_lo, s13
; GCN-HSA-NEXT: s_lshr_b32 flat_scratch_hi, s12, 8
-; GCN-HSA-NEXT: v_mov_b32_e32 v5, 0
+; GCN-HSA-NEXT: v_mov_b32_e32 v4, 0
; GCN-HSA-NEXT: s_waitcnt lgkmcnt(0)
; GCN-HSA-NEXT: v_mov_b32_e32 v0, s2
; GCN-HSA-NEXT: v_mov_b32_e32 v1, s3
; GCN-HSA-NEXT: flat_load_dwordx4 v[0:3], v[0:1]
-; GCN-HSA-NEXT: s_add_u32 s2, s0, 0x50
+; GCN-HSA-NEXT: s_add_u32 s2, s0, 0x70
; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
-; GCN-HSA-NEXT: s_add_u32 s4, s0, 0x70
+; GCN-HSA-NEXT: s_add_u32 s4, s0, 0x50
; GCN-HSA-NEXT: s_addc_u32 s5, s1, 0
-; GCN-HSA-NEXT: v_mov_b32_e32 v16, s5
-; GCN-HSA-NEXT: v_mov_b32_e32 v15, s4
-; GCN-HSA-NEXT: s_add_u32 s4, s0, 16
-; GCN-HSA-NEXT: v_mov_b32_e32 v9, v5
-; GCN-HSA-NEXT: v_mov_b32_e32 v11, v5
+; GCN-HSA-NEXT: v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT: v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT: s_add_u32 s4, s0, 48
+; GCN-HSA-NEXT: v_mov_b32_e32 v8, v4
+; GCN-HSA-NEXT: v_mov_b32_e32 v10, v4
; GCN-HSA-NEXT: s_addc_u32 s5, s1, 0
-; GCN-HSA-NEXT: v_mov_b32_e32 v12, v5
-; GCN-HSA-NEXT: v_mov_b32_e32 v14, v5
-; GCN-HSA-NEXT: v_mov_b32_e32 v17, v5
-; GCN-HSA-NEXT: v_mov_b32_e32 v7, v5
-; GCN-HSA-NEXT: v_mov_b32_e32 v18, v5
+; GCN-HSA-NEXT: v_mov_b32_e32 v11, v4
+; GCN-HSA-NEXT: v_mov_b32_e32 v13, v4
+; GCN-HSA-NEXT: v_mov_b32_e32 v16, v4
+; GCN-HSA-NEXT: v_mov_b32_e32 v6, v4
+; GCN-HSA-NEXT: v_mov_b32_e32 v17, v4
; GCN-HSA-NEXT: s_waitcnt vmcnt(0)
-; GCN-HSA-NEXT: v_lshrrev_b32_e32 v10, 24, v3
-; GCN-HSA-NEXT: v_bfe_u32 v8, v3, 16, 8
-; GCN-HSA-NEXT: flat_store_dwordx4 v[15:16], v[8:11]
-; GCN-HSA-NEXT: v_lshrrev_b32_e32 v13, 24, v0
-; GCN-HSA-NEXT: v_mov_b32_e32 v9, s5
-; GCN-HSA-NEXT: v_mov_b32_e32 v8, s4
-; GCN-HSA-NEXT: s_add_u32 s4, s0, 48
-; GCN-HSA-NEXT: v_bfe_u32 v11, v0, 16, 8
+; GCN-HSA-NEXT: v_lshrrev_b32_e32 v9, 24, v2
+; GCN-HSA-NEXT: v_bfe_u32 v7, v2, 16, 8
+; GCN-HSA-NEXT: flat_store_dwordx4 v[14:15], v[7:10]
+; GCN-HSA-NEXT: v_lshrrev_b32_e32 v12, 24, v1
+; GCN-HSA-NEXT: v_mov_b32_e32 v8, s5
+; GCN-HSA-NEXT: v_mov_b32_e32 v7, s4
+; GCN-HSA-NEXT: s_add_u32 s4, s0, 16
+; GCN-HSA-NEXT: v_bfe_u32 v10, v1, 16, 8
; GCN-HSA-NEXT: s_addc_u32 s5, s1, 0
-; GCN-HSA-NEXT: flat_store_dwordx4 v[8:9], v[11:14]
-; GCN-HSA-NEXT: v_mov_b32_e32 v15, v5
-; GCN-HSA-NEXT: v_mov_b32_e32 v13, s5
-; GCN-HSA-NEXT: v_mov_b32_e32 v12, s4
-; GCN-HSA-NEXT: s_add_u32 s4, s0, 64
-; GCN-HSA-NEXT: v_lshrrev_b32_e32 v16, 24, v1
-; GCN-HSA-NEXT: v_bfe_u32 v14, v1, 16, 8
+; GCN-HSA-NEXT: flat_store_dwordx4 v[7:8], v[10:13]
+; GCN-HSA-NEXT: v_mov_b32_e32 v14, v4
+; GCN-HSA-NEXT: v_mov_b32_e32 v12, s5
+; GCN-HSA-NEXT: v_mov_b32_e32 v11, s4
+; GCN-HSA-NEXT: s_add_u32 s4, s0, 0x60
+; GCN-HSA-NEXT: v_lshrrev_b32_e32 v15, 24, v0
+; GCN-HSA-NEXT: v_bfe_u32 v13, v0, 16, 8
; GCN-HSA-NEXT: s_addc_u32 s5, s1, 0
-; GCN-HSA-NEXT: v_mov_b32_e32 v9, v5
-; GCN-HSA-NEXT: v_mov_b32_e32 v11, v5
-; GCN-HSA-NEXT: flat_store_dwordx4 v[12:13], v[14:17]
-; GCN-HSA-NEXT: v_bfe_u32 v10, v2, 8, 8
-; GCN-HSA-NEXT: v_mov_b32_e32 v16, s5
-; GCN-HSA-NEXT: v_mov_b32_e32 v15, s4
-; GCN-HSA-NEXT: v_and_b32_e32 v8, 0xff, v2
-; GCN-HSA-NEXT: flat_store_dwordx4 v[15:16], v[8:11]
-; GCN-HSA-NEXT: v_mov_b32_e32 v16, s3
-; GCN-HSA-NEXT: v_mov_b32_e32 v15, s2
-; GCN-HSA-NEXT: s_add_u32 s2, s0, 0x60
-; GCN-HSA-NEXT: v_lshrrev_b32_e32 v6, 24, v2
-; GCN-HSA-NEXT: v_bfe_u32 v4, v2, 16, 8
+; GCN-HSA-NEXT: v_mov_b32_e32 v8, v4
+; GCN-HSA-NEXT: v_mov_b32_e32 v10, v4
+; GCN-HSA-NEXT: flat_store_dwordx4 v[11:12], v[13:16]
+; GCN-HSA-NEXT: v_bfe_u32 v9, v3, 8, 8
+; GCN-HSA-NEXT: v_mov_b32_e32 v15, s5
+; GCN-HSA-NEXT: v_mov_b32_e32 v14, s4
+; GCN-HSA-NEXT: v_and_b32_e32 v7, 0xff, v3
+; GCN-HSA-NEXT: flat_store_dwordx4 v[14:15], v[7:10]
+; GCN-HSA-NEXT: v_mov_b32_e32 v15, s3
+; GCN-HSA-NEXT: v_mov_b32_e32 v14, s2
+; GCN-HSA-NEXT: s_add_u32 s2, s0, 64
+; GCN-HSA-NEXT: v_lshrrev_b32_e32 v5, 24, v3
+; GCN-HSA-NEXT: v_bfe_u32 v3, v3, 16, 8
; GCN-HSA-NEXT: s_addc_u32 s3, s1, 0
-; GCN-HSA-NEXT: v_mov_b32_e32 v12, v5
-; GCN-HSA-NEXT: v_mov_b32_e32 v14, v5
-; GCN-HSA-NEXT: v_mov_b32_e32 v8, v5
-; GCN-HSA-NEXT: v_mov_b32_e32 v10, v5
-; GCN-HSA-NEXT: flat_store_dwordx4 v[15:16], v[4:7]
-; GCN-HSA-NEXT: v_mov_b32_e32 v16, v5
-; GCN-HSA-NEXT: v_mov_b32_e32 v5, s1
-; GCN-HSA-NEXT: v_mov_b32_e32 v4, s0
+; GCN-HSA-NEXT: v_mov_b32_e32 v11, v4
+; GCN-HSA-NEXT: v_mov_b32_e32 v13, v4
+; GCN-HSA-NEXT: v_mov_b32_e32 v7, v4
+; GCN-HSA-NEXT: v_mov_b32_e32 v9, v4
+; GCN-HSA-NEXT: flat_store_dwordx4 v[14:15], v[3:6]
+; GCN-HSA-NEXT: v_mov_b32_e32 v15, v4
+; GCN-HSA-NEXT: v_mov_b32_e32 v4, s1
+; GCN-HSA-NEXT: v_mov_b32_e32 v3, s0
; GCN-HSA-NEXT: s_add_u32 s0, s0, 32
-; GCN-HSA-NEXT: v_mov_b32_e32 v20, s3
-; GCN-HSA-NEXT: v_mov_b32_e32 v19, s2
+; GCN-HSA-NEXT: v_mov_b32_e32 v19, s3
+; GCN-HSA-NEXT: v_mov_b32_e32 v18, s2
; GCN-HSA-NEXT: s_addc_u32 s1, s1, 0
-; GCN-HSA-NEXT: v_bfe_u32 v13, v3, 8, 8
-; GCN-HSA-NEXT: v_and_b32_e32 v11, 0xff, v3
-; GCN-HSA-NEXT: v_bfe_u32 v9, v0, 8, 8
-; GCN-HSA-NEXT: v_bfe_u32 v17, v1, 8, 8
-; GCN-HSA-NEXT: v_and_b32_e32 v15, 0xff, v1
-; GCN-HSA-NEXT: v_and_b32_e32 v7, 0xff, v0
+; GCN-HSA-NEXT: v_bfe_u32 v12, v2, 8, 8
+; GCN-HSA-NEXT: v_and_b32_e32 v10, 0xff, v2
+; GCN-HSA-NEXT: v_bfe_u32 v8, v1, 8, 8
+; GCN-HSA-NEXT: v_bfe_u32 v16, v0, 8, 8
+; GCN-HSA-NEXT: v_and_b32_e32 v14, 0xff, v0
+; GCN-HSA-NEXT: v_and_b32_e32 v6, 0xff, v1
; GCN-HSA-NEXT: v_mov_b32_e32 v0, s0
; GCN-HSA-NEXT: v_mov_b32_e32 v1, s1
-; GCN-HSA-NEXT: flat_store_dwordx4 v[19:20], v[11:14]
-; GCN-HSA-NEXT: flat_store_dwordx4 v[4:5], v[7:10]
-; GCN-HSA-NEXT: flat_store_dwordx4 v[0:1], v[15:18]
+; GCN-HSA-NEXT: flat_store_dwordx4 v[18:19], v[10:13]
+; GCN-HSA-NEXT: flat_store_dwordx4 v[0:1], v[6:9]
+; GCN-HSA-NEXT: flat_store_dwordx4 v[3:4], v[14:17]
; GCN-HSA-NEXT: s_endpgm
;
; GCN-NOHSA-VI-LABEL: global_zextload_v16i8_to_v16i64:
@@ -6515,21 +6515,21 @@ define amdgpu_kernel void @global_zextload_v16i8_to_v16i64(ptr addrspace(1) %out
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v12, 0xff, v0
; GCN-NOHSA-VI-NEXT: v_bfe_u32 v8, v0, 16, 8
; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v4, 0xff, v1
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v18, v3, 8, 8
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v16, 0xff, v3
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v22, v2, 8, 8
-; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v20, 0xff, v2
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v26, 24, v2
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v24, v2, 16, 8
-; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v2, 24, v1
-; GCN-NOHSA-VI-NEXT: v_bfe_u32 v0, v1, 16, 8
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v18, v2, 8, 8
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v16, 0xff, v2
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v22, 24, v2
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v20, v2, 16, 8
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v2, v3, 8, 8
+; GCN-NOHSA-VI-NEXT: v_and_b32_e32 v0, 0xff, v3
+; GCN-NOHSA-VI-NEXT: v_lshrrev_b32_e32 v26, 24, v1
+; GCN-NOHSA-VI-NEXT: v_bfe_u32 v24, v1, 16, 8
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v1, v29
; GCN-NOHSA-VI-NEXT: v_mov_b32_e32 v3, v29
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[28:31], off, s[0:3], 0 offset:112
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:48
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:80
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:64
-; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[24:27], off, s[0:3], 0 offset:48
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[0:3], off, s[0:3], 0 offset:96
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[20:23], off, s[0:3], 0 offset:80
+; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[16:19], off, s[0:3], 0 offset:64
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[4:7], off, s[0:3], 0 offset:32
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[8:11], off, s[0:3], 0 offset:16
; GCN-NOHSA-VI-NEXT: buffer_store_dwordx4 v[12:15], off, s[0:3], 0
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
index 8301e1b3e7f97a..6f885d73b61dfa 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
@@ -2119,112 +2119,110 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NO-DS128-NEXT: s_mov_b32 m0, -1
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v24, s1
-; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v24 offset1:1
-; VI-NO-DS128-NEXT: ds_read2_b64 v[8:11], v24 offset0:2 offset1:3
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v10, s1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[2:5], v10 offset1:1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[6:9], v10 offset0:2 offset1:3
; VI-NO-DS128-NEXT: v_mov_b32_e32 v32, s0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v7
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; VI-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v6
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v5
; VI-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v5
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v4
; VI-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v4
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v11
-; VI-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v11
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v10
-; VI-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v10
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v7
+; VI-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v7
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v6
+; VI-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v6
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v9
-; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v24 offset0:4 offset1:5
+; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v10 offset0:4 offset1:5
; VI-NO-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v9
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v8
; VI-NO-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v8
-; VI-NO-DS128-NEXT: ds_read2_b64 v[8:11], v24 offset0:6 offset1:7
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; VI-NO-DS128-NEXT: ds_read2_b64 v[8:11], v10 offset0:6 offset1:7
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; VI-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v11
+; VI-NO-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v11
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v10
+; VI-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v10
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v5
; VI-NO-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v5
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v9
-; VI-NO-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v9
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; VI-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v8
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v27, 16, v7
; VI-NO-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v7
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v11
-; VI-NO-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v11
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; VI-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[8:9], v[30:31] offset0:12 offset1:13
-; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[10:11], v[28:29] offset0:14 offset1:15
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v9
+; VI-NO-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v9
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v8
+; VI-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[10:11], v[30:31] offset0:14 offset1:15
+; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[8:9], v[28:29] offset0:12 offset1:13
; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[6:7], v[26:27] offset0:10 offset1:11
; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[4:5], v[24:25] offset0:8 offset1:9
-; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[22:23], v[20:21] offset0:4 offset1:5
-; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[18:19], v[16:17] offset0:6 offset1:7
-; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[14:15], v[12:13] offset1:1
-; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[2:3], v[0:1] offset0:2 offset1:3
+; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[22:23], v[20:21] offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[18:19], v[16:17] offset0:4 offset1:5
+; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[14:15], v[12:13] offset0:2 offset1:3
+; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[2:3], v[0:1] offset1:1
; VI-NO-DS128-NEXT: s_endpgm
;
; GFX9-NO-DS128-LABEL: local_zextload_v32i16_to_v32i32:
; GFX9-NO-DS128: ; %bb.0:
; GFX9-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v24, s1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v24 offset1:1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[8:11], v24 offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v10, s1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[2:5], v10 offset1:1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[6:9], v10 offset0:2 offset1:3
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v32, s0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v7
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v7
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v6
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v6
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v5
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v5
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v4
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v4
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v11
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v11
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v10
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v10
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v7
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v7
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v6
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v6
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v9
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v24 offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v10 offset0:4 offset1:5
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v9
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v8
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v8
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[8:11], v24 offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[8:11], v10 offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v3
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v11
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v11
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v10
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v10
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v5
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v5
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v9
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v9
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v8
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v8
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v27, 16, v7
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v7
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v11
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v11
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v10
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v10
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[8:9], v[30:31] offset0:12 offset1:13
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[10:11], v[28:29] offset0:14 offset1:15
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v9
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v9
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v8
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[10:11], v[30:31] offset0:14 offset1:15
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[8:9], v[28:29] offset0:12 offset1:13
; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[6:7], v[26:27] offset0:10 offset1:11
; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[4:5], v[24:25] offset0:8 offset1:9
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[22:23], v[20:21] offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[18:19], v[16:17] offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[14:15], v[12:13] offset1:1
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[2:3], v[0:1] offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[22:23], v[20:21] offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[18:19], v[16:17] offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[14:15], v[12:13] offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[2:3], v[0:1] offset1:1
; GFX9-NO-DS128-NEXT: s_endpgm
;
; EG-LABEL: local_zextload_v32i16_to_v32i32:
@@ -2608,24 +2606,24 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: ds_read2_b64 v[12:15], v24 offset0:2 offset1:3
; VI-NO-DS128-NEXT: v_mov_b32_e32 v32, s0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v11
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v10
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v9
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v8
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v9
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v8
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v11
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v10
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v15
-; VI-NO-DS128-NEXT: v_bfe_i32 v0, v11, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v2, v10, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v4, v9, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v6, v8, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v14
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v21, 16, v13
-; VI-NO-DS128-NEXT: v_bfe_i32 v16, v15, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v18, v14, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v13
+; VI-NO-DS128-NEXT: v_bfe_i32 v0, v9, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v2, v8, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v4, v11, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v6, v10, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v12
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v21, 16, v15
+; VI-NO-DS128-NEXT: v_bfe_i32 v16, v13, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v18, v12, 0, 16
; VI-NO-DS128-NEXT: ds_read2_b64 v[8:11], v24 offset0:4 offset1:5
-; VI-NO-DS128-NEXT: v_bfe_i32 v20, v13, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v23, 16, v12
-; VI-NO-DS128-NEXT: v_bfe_i32 v22, v12, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v20, v15, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v23, 16, v14
+; VI-NO-DS128-NEXT: v_bfe_i32 v22, v14, 0, 16
; VI-NO-DS128-NEXT: ds_read2_b64 v[12:15], v24 offset0:6 offset1:7
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v25, 16, v9
@@ -2633,26 +2631,26 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v8
; VI-NO-DS128-NEXT: v_bfe_i32 v8, v8, 0, 16
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v31, 16, v13
-; VI-NO-DS128-NEXT: v_bfe_i32 v30, v13, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 16, v12
-; VI-NO-DS128-NEXT: v_bfe_i32 v12, v12, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v31, 16, v15
+; VI-NO-DS128-NEXT: v_bfe_i32 v30, v15, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v14
+; VI-NO-DS128-NEXT: v_bfe_i32 v14, v14, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v27, 16, v11
; VI-NO-DS128-NEXT: v_bfe_i32 v26, v11, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v10
; VI-NO-DS128-NEXT: v_bfe_i32 v10, v10, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v29, 16, v15
-; VI-NO-DS128-NEXT: v_bfe_i32 v28, v15, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v14
-; VI-NO-DS128-NEXT: v_bfe_i32 v14, v14, 0, 16
-; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[12:13], v[30:31] offset0:12 offset1:13
-; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[14:15], v[28:29] offset0:14 offset1:15
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v29, 16, v13
+; VI-NO-DS128-NEXT: v_bfe_i32 v28, v13, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 16, v12
+; VI-NO-DS128-NEXT: v_bfe_i32 v12, v12, 0, 16
+; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[14:15], v[30:31] offset0:14 offset1:15
+; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[12:13], v[28:29] offset0:12 offset1:13
; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[10:11], v[26:27] offset0:10 offset1:11
; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[8:9], v[24:25] offset0:8 offset1:9
-; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[22:23], v[20:21] offset0:4 offset1:5
-; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[18:19], v[16:17] offset0:6 offset1:7
-; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[6:7], v[4:5] offset1:1
-; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[2:3], v[0:1] offset0:2 offset1:3
+; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[22:23], v[20:21] offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[18:19], v[16:17] offset0:4 offset1:5
+; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[6:7], v[4:5] offset0:2 offset1:3
+; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[2:3], v[0:1] offset1:1
; VI-NO-DS128-NEXT: s_endpgm
;
; GFX9-NO-DS128-LABEL: local_sextload_v32i16_to_v32i32:
@@ -2664,24 +2662,24 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[12:15], v24 offset0:2 offset1:3
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v32, s0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v11
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v10
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v9
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v8
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v9
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v8
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v11
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v10
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v15
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v11, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v10, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v9, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v8, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v14
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v21, 16, v13
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v15, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v14, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v13
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v9, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v8, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v11, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v10, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v12
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v21, 16, v15
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v13, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v12, 0, 16
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[8:11], v24 offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v20, v13, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v23, 16, v12
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v22, v12, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v20, v15, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v23, 16, v14
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v22, v14, 0, 16
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[12:15], v24 offset0:6 offset1:7
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v25, 16, v9
@@ -2689,26 +2687,26 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v8
; GFX9-NO-DS128-NEXT: v_bfe_i32 v8, v8, 0, 16
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v31, 16, v13
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v30, v13, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 16, v12
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v12, v12, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v31, 16, v15
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v30, v15, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v14
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v14, v14, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v27, 16, v11
; GFX9-NO-DS128-NEXT: v_bfe_i32 v26, v11, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v10
; GFX9-NO-DS128-NEXT: v_bfe_i32 v10, v10, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v29, 16, v15
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v28, v15, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v14
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v14, v14, 0, 16
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[12:13], v[30:31] offset0:12 offset1:13
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[14:15], v[28:29] offset0:14 offset1:15
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v29, 16, v13
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v28, v13, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 16, v12
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v12, v12, 0, 16
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[14:15], v[30:31] offset0:14 offset1:15
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[12:13], v[28:29] offset0:12 offset1:13
; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[10:11], v[26:27] offset0:10 offset1:11
; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[8:9], v[24:25] offset0:8 offset1:9
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[22:23], v[20:21] offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[18:19], v[16:17] offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[6:7], v[4:5] offset1:1
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[2:3], v[0:1] offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[22:23], v[20:21] offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[18:19], v[16:17] offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[6:7], v[4:5] offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[2:3], v[0:1] offset1:1
; GFX9-NO-DS128-NEXT: s_endpgm
;
; EG-LABEL: local_sextload_v32i16_to_v32i32:
@@ -5904,30 +5902,30 @@ define amdgpu_kernel void @local_zextload_v8i16_to_v8i64(ptr addrspace(3) %out,
; VI-NO-DS128: ; %bb.0:
; VI-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NO-DS128-NEXT: s_mov_b32 m0, -1
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v10, 0
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v8, v10
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-NO-DS128-NEXT: v_mov_b32_e32 v0, s1
; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v0 offset1:1
; VI-NO-DS128-NEXT: v_mov_b32_e32 v11, s0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v3
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v3, 0
-; VI-NO-DS128-NEXT: v_and_b32_e32 v9, 0xffff, v2
+; VI-NO-DS128-NEXT: v_and_b32_e32 v7, 0xffff, v3
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v3
+; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v2
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v10, v3
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[9:10], v[2:3] offset0:4 offset1:5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v9, v3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v7, v3
+; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[7:8], v[9:10] offset0:6 offset1:7
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v3, v10
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v7, v10
; VI-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v1
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[6:7], v[8:9] offset0:6 offset1:7
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v2, v3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v6, v3
+; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[6:7], v[2:3] offset0:4 offset1:5
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v2, v10
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v6, v10
; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v0
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[5:6], v[1:2] offset0:2 offset1:3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v1, v3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v5, v3
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v1, v10
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v5, v10
; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[4:5], v[0:1] offset1:1
; VI-NO-DS128-NEXT: s_endpgm
;
@@ -6387,131 +6385,131 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i64(ptr addrspace(3) %out
; SI-NEXT: s_waitcnt lgkmcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v1
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v5
-; SI-NEXT: v_and_b32_e32 v16, 0xffff, v5
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v7
-; SI-NEXT: ds_write2_b64 v18, v[16:17], v[14:15] offset0:10 offset1:11
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v7
+; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v7
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v7
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v5
+; SI-NEXT: ds_write2_b64 v18, v[16:17], v[14:15] offset0:14 offset1:15
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v5
; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v2
-; SI-NEXT: ds_write2_b64 v18, v[14:15], v[12:13] offset0:14 offset1:15
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v2
+; SI-NEXT: ds_write2_b64 v18, v[14:15], v[12:13] offset0:10 offset1:11
; SI-NEXT: v_and_b32_e32 v12, 0xffff, v3
; SI-NEXT: v_and_b32_e32 v1, 0xffff, v1
; SI-NEXT: v_and_b32_e32 v3, 0xffff, v2
; SI-NEXT: v_mov_b32_e32 v2, v9
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; SI-NEXT: ds_write2_b64 v18, v[12:13], v[10:11] offset0:6 offset1:7
; SI-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v4
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v6
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v6
-; SI-NEXT: v_and_b32_e32 v16, 0xffff, v4
+; SI-NEXT: v_lshrrev_b32_e32 v10, 16, v6
+; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v4
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v4
+; SI-NEXT: v_and_b32_e32 v16, 0xffff, v6
; SI-NEXT: ds_write2_b64 v18, v[1:2], v[8:9] offset0:2 offset1:3
; SI-NEXT: v_mov_b32_e32 v4, v9
; SI-NEXT: v_mov_b32_e32 v1, v9
-; SI-NEXT: v_mov_b32_e32 v6, v9
; SI-NEXT: v_mov_b32_e32 v8, v9
-; SI-NEXT: ds_write2_b64 v18, v[16:17], v[10:11] offset0:8 offset1:9
-; SI-NEXT: ds_write2_b64 v18, v[14:15], v[12:13] offset0:12 offset1:13
-; SI-NEXT: ds_write2_b64 v18, v[3:4], v[5:6] offset0:4 offset1:5
-; SI-NEXT: ds_write2_b64 v18, v[0:1], v[7:8] offset1:1
+; SI-NEXT: v_mov_b32_e32 v6, v9
+; SI-NEXT: ds_write2_b64 v18, v[16:17], v[10:11] offset0:12 offset1:13
+; SI-NEXT: ds_write2_b64 v18, v[14:15], v[12:13] offset0:8 offset1:9
+; SI-NEXT: ds_write2_b64 v18, v[3:4], v[7:8] offset0:4 offset1:5
+; SI-NEXT: ds_write2_b64 v18, v[0:1], v[5:6] offset1:1
; SI-NEXT: s_endpgm
;
; VI-NO-DS128-LABEL: local_zextload_v16i16_to_v16i64:
; VI-NO-DS128: ; %bb.0:
; VI-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NO-DS128-NEXT: s_mov_b32 m0, -1
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v8, 0
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v10, v8
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v13, v8
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v9, 0
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v11, v9
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v14, v9
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset1:1
; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v4 offset0:2 offset1:3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v14, s0
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v15, s0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v0
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v12, 16, v0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v5
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[12:13], v[9:10] offset0:10 offset1:11
-; VI-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v6
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v6
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v6, v8
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[5:6], v[9:10] offset0:12 offset1:13
-; VI-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v7
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v7
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[5:6], v[9:10] offset0:14 offset1:15
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v3
-; VI-NO-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v4
-; VI-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v4
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, v8
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v6
+; VI-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v6
+; VI-NO-DS128-NEXT: ds_write2_b64 v15, v[10:11], v[13:14] offset0:12 offset1:13
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; VI-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v6, v9
+; VI-NO-DS128-NEXT: ds_write2_b64 v15, v[5:6], v[10:11] offset0:10 offset1:11
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; VI-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v4
+; VI-NO-DS128-NEXT: ds_write2_b64 v15, v[10:11], v[5:6] offset0:8 offset1:9
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; VI-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v3
+; VI-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v7
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v7
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v7, v9
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; VI-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[12:13], v[7:8] offset0:8 offset1:9
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[3:4], v[9:10] offset0:6 offset1:7
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v3, v8
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v7, v8
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; VI-NO-DS128-NEXT: ds_write2_b64 v15, v[10:11], v[6:7] offset0:6 offset1:7
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v3, v9
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v6, v9
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; VI-NO-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[2:3], v[6:7] offset0:4 offset1:5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v2, v8
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v6, v8
+; VI-NO-DS128-NEXT: ds_write2_b64 v15, v[2:3], v[5:6] offset0:4 offset1:5
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v2, v9
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v5, v9
; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[1:2], v[5:6] offset0:2 offset1:3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v1, v8
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v12, v8
-; VI-NO-DS128-NEXT: ds_write2_b64 v14, v[0:1], v[11:12] offset1:1
+; VI-NO-DS128-NEXT: ds_write2_b64 v15, v[8:9], v[13:14] offset0:14 offset1:15
+; VI-NO-DS128-NEXT: ds_write2_b64 v15, v[1:2], v[4:5] offset0:2 offset1:3
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v1, v9
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v13, v9
+; VI-NO-DS128-NEXT: ds_write2_b64 v15, v[0:1], v[12:13] offset1:1
; VI-NO-DS128-NEXT: s_endpgm
;
; GFX9-NO-DS128-LABEL: local_zextload_v16i16_to_v16i64:
; GFX9-NO-DS128: ; %bb.0:
; GFX9-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v8, 0
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v10, v8
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v12, v8
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v14, v8
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v9, 0
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v13, v9
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v15, v9
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset1:1
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v4 offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v15, s0
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v16, s0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v0
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v5
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[11:12], v[9:10] offset0:10 offset1:11
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v6
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v6
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v6, v8
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[5:6], v[9:10] offset0:12 offset1:13
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v7
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v7
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[5:6], v[9:10] offset0:14 offset1:15
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v4
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v12, 16, v6
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v6
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[10:11], v[12:13] offset0:12 offset1:13
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v5
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v5
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v6, v9
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[5:6], v[10:11] offset0:10 offset1:11
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v4
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v11, v8
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v3
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[10:11], v[7:8] offset0:8 offset1:9
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v4, v8
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v10, v8
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v2
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[10:11], v[5:6] offset0:8 offset1:9
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v7
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v7
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v12, v9
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v3
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v3
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[8:9], v[11:12] offset0:14 offset1:15
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v11, v9
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v7, v9
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v2
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[3:4], v[9:10] offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v3, v8
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v7, v8
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[10:11], v[6:7] offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v3, v9
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v6, v9
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v1
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[2:3], v[6:7] offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v2, v8
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v6, v8
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[2:3], v[5:6] offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v2, v9
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v5, v9
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[1:2], v[5:6] offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v1, v8
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[0:1], v[13:14] offset1:1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[1:2], v[4:5] offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v1, v9
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[0:1], v[14:15] offset1:1
; GFX9-NO-DS128-NEXT: s_endpgm
;
; EG-LABEL: local_zextload_v16i16_to_v16i64:
@@ -6668,53 +6666,53 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i64(ptr addrspace(3) %out
; VI-DS128: ; %bb.0:
; VI-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-DS128-NEXT: s_mov_b32 m0, -1
-; VI-DS128-NEXT: v_mov_b32_e32 v26, 0
-; VI-DS128-NEXT: v_mov_b32_e32 v22, v26
-; VI-DS128-NEXT: v_mov_b32_e32 v24, v26
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-DS128-NEXT: v_mov_b32_e32 v5, s1
; VI-DS128-NEXT: ds_read_b128 v[0:3], v5
; VI-DS128-NEXT: ds_read_b128 v[13:16], v5 offset:16
-; VI-DS128-NEXT: v_mov_b32_e32 v11, v26
-; VI-DS128-NEXT: v_mov_b32_e32 v19, v26
-; VI-DS128-NEXT: v_mov_b32_e32 v8, v26
+; VI-DS128-NEXT: v_mov_b32_e32 v27, s0
; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
; VI-DS128-NEXT: v_lshrrev_b32_e32 v12, 16, v2
-; VI-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v2
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v13
-; VI-DS128-NEXT: v_and_b32_e32 v21, 0xffff, v13
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v27, 16, v14
-; VI-DS128-NEXT: v_and_b32_e32 v25, 0xffff, v14
-; VI-DS128-NEXT: v_mov_b32_e32 v14, s0
-; VI-DS128-NEXT: v_mov_b32_e32 v13, v26
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v14
+; VI-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v14
+; VI-DS128-NEXT: v_mov_b32_e32 v14, 0
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v13
+; VI-DS128-NEXT: v_and_b32_e32 v17, 0xffff, v13
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v15
+; VI-DS128-NEXT: v_and_b32_e32 v23, 0xffff, v15
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v16
+; VI-DS128-NEXT: v_and_b32_e32 v13, 0xffff, v16
+; VI-DS128-NEXT: v_mov_b32_e32 v16, v14
+; VI-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v2
+; VI-DS128-NEXT: v_mov_b32_e32 v24, v14
+; VI-DS128-NEXT: v_mov_b32_e32 v26, v14
+; VI-DS128-NEXT: ds_write_b128 v27, v[13:16] offset:112
+; VI-DS128-NEXT: v_mov_b32_e32 v11, v14
+; VI-DS128-NEXT: v_mov_b32_e32 v13, v14
; VI-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v1
; VI-DS128-NEXT: v_and_b32_e32 v7, 0xffff, v1
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v16
-; VI-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v16
-; VI-DS128-NEXT: ds_write_b128 v14, v[21:24] offset:64
-; VI-DS128-NEXT: v_mov_b32_e32 v21, v26
-; VI-DS128-NEXT: ds_write_b128 v14, v[10:13] offset:32
-; VI-DS128-NEXT: v_mov_b32_e32 v10, v26
+; VI-DS128-NEXT: ds_write_b128 v27, v[23:26] offset:96
+; VI-DS128-NEXT: v_mov_b32_e32 v21, v14
+; VI-DS128-NEXT: v_mov_b32_e32 v23, v14
+; VI-DS128-NEXT: ds_write_b128 v27, v[10:13] offset:32
+; VI-DS128-NEXT: v_mov_b32_e32 v8, v14
+; VI-DS128-NEXT: v_mov_b32_e32 v10, v14
; VI-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v0
; VI-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v0
; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; VI-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v15
-; VI-DS128-NEXT: v_and_b32_e32 v15, 0xffff, v15
-; VI-DS128-NEXT: ds_write_b128 v14, v[18:21] offset:112
-; VI-DS128-NEXT: v_mov_b32_e32 v16, v26
-; VI-DS128-NEXT: v_mov_b32_e32 v18, v26
-; VI-DS128-NEXT: v_mov_b32_e32 v1, v26
-; VI-DS128-NEXT: v_mov_b32_e32 v3, v26
-; VI-DS128-NEXT: v_mov_b32_e32 v28, v26
-; VI-DS128-NEXT: ds_write_b128 v14, v[7:10] offset:16
-; VI-DS128-NEXT: v_mov_b32_e32 v5, v26
-; VI-DS128-NEXT: v_mov_b32_e32 v7, v26
-; VI-DS128-NEXT: ds_write_b128 v14, v[15:18] offset:96
-; VI-DS128-NEXT: ds_write_b128 v14, v[0:3] offset:48
-; VI-DS128-NEXT: ds_write_b128 v14, v[25:28] offset:80
-; VI-DS128-NEXT: ds_write_b128 v14, v[4:7]
+; VI-DS128-NEXT: ds_write_b128 v27, v[20:23] offset:80
+; VI-DS128-NEXT: v_mov_b32_e32 v18, v14
+; VI-DS128-NEXT: v_mov_b32_e32 v20, v14
+; VI-DS128-NEXT: v_mov_b32_e32 v1, v14
+; VI-DS128-NEXT: v_mov_b32_e32 v3, v14
+; VI-DS128-NEXT: ds_write_b128 v27, v[7:10] offset:16
+; VI-DS128-NEXT: v_mov_b32_e32 v5, v14
+; VI-DS128-NEXT: v_mov_b32_e32 v7, v14
+; VI-DS128-NEXT: ds_write_b128 v27, v[17:20] offset:64
+; VI-DS128-NEXT: ds_write_b128 v27, v[0:3] offset:48
+; VI-DS128-NEXT: ds_write_b128 v27, v[4:7]
; VI-DS128-NEXT: s_endpgm
;
; GFX9-DS128-LABEL: local_zextload_v16i16_to_v16i64:
@@ -6728,45 +6726,45 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i64(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_mov_b32_e32 v4, s1
; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v4
; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v4 offset:16
-; GFX9-DS128-NEXT: v_mov_b32_e32 v28, s0
; GFX9-DS128-NEXT: v_mov_b32_e32 v15, v25
; GFX9-DS128-NEXT: v_mov_b32_e32 v12, v25
+; GFX9-DS128-NEXT: v_mov_b32_e32 v27, v25
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v2
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v7
-; GFX9-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v7
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v6
-; GFX9-DS128-NEXT: v_and_b32_e32 v17, 0xffff, v6
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[20:23] offset:112
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v5
+; GFX9-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v5
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v26, 16, v7
+; GFX9-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v7
+; GFX9-DS128-NEXT: v_mov_b32_e32 v7, s0
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v4
+; GFX9-DS128-NEXT: v_and_b32_e32 v17, 0xffff, v4
+; GFX9-DS128-NEXT: ds_write_b128 v7, v[20:23] offset:80
; GFX9-DS128-NEXT: v_mov_b32_e32 v20, v25
; GFX9-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v2
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[17:20] offset:96
+; GFX9-DS128-NEXT: ds_write_b128 v7, v[17:20] offset:64
; GFX9-DS128-NEXT: v_mov_b32_e32 v17, v25
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v1
-; GFX9-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v1
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[14:17] offset:32
-; GFX9-DS128-NEXT: v_mov_b32_e32 v14, v25
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v0
; GFX9-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v1
+; GFX9-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v1
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX9-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX9-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v26, 16, v5
-; GFX9-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v5
-; GFX9-DS128-NEXT: v_mov_b32_e32 v5, v25
-; GFX9-DS128-NEXT: v_mov_b32_e32 v7, v25
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v6
+; GFX9-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v6
+; GFX9-DS128-NEXT: v_mov_b32_e32 v4, v25
+; GFX9-DS128-NEXT: v_mov_b32_e32 v6, v25
+; GFX9-DS128-NEXT: ds_write_b128 v7, v[14:17] offset:32
+; GFX9-DS128-NEXT: v_mov_b32_e32 v14, v25
+; GFX9-DS128-NEXT: ds_write_b128 v7, v[3:6] offset:96
; GFX9-DS128-NEXT: v_mov_b32_e32 v1, v25
; GFX9-DS128-NEXT: v_mov_b32_e32 v3, v25
-; GFX9-DS128-NEXT: v_mov_b32_e32 v27, v25
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[11:14] offset:16
+; GFX9-DS128-NEXT: ds_write_b128 v7, v[11:14] offset:16
; GFX9-DS128-NEXT: v_mov_b32_e32 v9, v25
; GFX9-DS128-NEXT: v_mov_b32_e32 v11, v25
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[4:7] offset:64
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[0:3] offset:48
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[24:27] offset:80
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[8:11]
+; GFX9-DS128-NEXT: ds_write_b128 v7, v[0:3] offset:48
+; GFX9-DS128-NEXT: ds_write_b128 v7, v[24:27] offset:112
+; GFX9-DS128-NEXT: ds_write_b128 v7, v[8:11]
; GFX9-DS128-NEXT: s_endpgm
%load = load <16 x i16>, ptr addrspace(3) %in
%ext = zext <16 x i16> %load to <16 x i64>
@@ -6840,57 +6838,57 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i64(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset1:1
; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v4 offset0:2 offset1:3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v19, s0
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v20, s0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v3
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v4
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v7
; VI-NO-DS128-NEXT: v_bfe_i32 v15, v15, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v17, v4, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v17, v7, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[17:18], v[15:16] offset0:8 offset1:9
-; VI-NO-DS128-NEXT: v_bfe_i32 v15, v4, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v4, v5, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; VI-NO-DS128-NEXT: ds_write2_b64 v20, v[17:18], v[15:16] offset0:14 offset1:15
+; VI-NO-DS128-NEXT: v_bfe_i32 v15, v7, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v6, v6, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[4:5], v[15:16] offset0:10 offset1:11
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; VI-NO-DS128-NEXT: v_bfe_i32 v4, v4, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v15, v6, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; VI-NO-DS128-NEXT: ds_write2_b64 v20, v[6:7], v[15:16] offset0:12 offset1:13
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; VI-NO-DS128-NEXT: v_bfe_i32 v6, v6, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v15, v5, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[15:16], v[4:5] offset0:12 offset1:13
-; VI-NO-DS128-NEXT: v_bfe_i32 v4, v14, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v14, v6, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
+; VI-NO-DS128-NEXT: ds_write2_b64 v20, v[15:16], v[6:7] offset0:10 offset1:11
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v4
+; VI-NO-DS128-NEXT: v_bfe_i32 v5, v14, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v14, v7, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v1
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v2
-; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[6:7], v[14:15] offset0:14 offset1:15
-; VI-NO-DS128-NEXT: v_bfe_i32 v6, v2, 0, 16
+; VI-NO-DS128-NEXT: ds_write2_b64 v20, v[16:17], v[14:15] offset0:8 offset1:9
+; VI-NO-DS128-NEXT: v_bfe_i32 v14, v2, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v2, v3, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v8, v8, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v10, v9, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v12, v11, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; VI-NO-DS128-NEXT: v_bfe_i32 v16, v0, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; VI-NO-DS128-NEXT: v_bfe_i32 v18, v0, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v0, v1, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[2:3], v[4:5] offset0:6 offset1:7
-; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[6:7], v[12:13] offset0:4 offset1:5
-; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[0:1], v[10:11] offset0:2 offset1:3
-; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[16:17], v[8:9] offset1:1
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; VI-NO-DS128-NEXT: ds_write2_b64 v20, v[2:3], v[5:6] offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_write2_b64 v20, v[14:15], v[12:13] offset0:4 offset1:5
+; VI-NO-DS128-NEXT: ds_write2_b64 v20, v[0:1], v[10:11] offset0:2 offset1:3
+; VI-NO-DS128-NEXT: ds_write2_b64 v20, v[18:19], v[8:9] offset1:1
; VI-NO-DS128-NEXT: s_endpgm
;
; GFX9-NO-DS128-LABEL: local_sextload_v16i16_to_v16i64:
@@ -6900,57 +6898,57 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i64(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset1:1
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v4 offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v19, s0
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v20, s0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v3
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v4
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v7
; GFX9-NO-DS128-NEXT: v_bfe_i32 v15, v15, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v17, v4, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v17, v7, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[17:18], v[15:16] offset0:8 offset1:9
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v15, v4, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v5, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[17:18], v[15:16] offset0:14 offset1:15
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v15, v7, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v6, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[4:5], v[15:16] offset0:10 offset1:11
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v4, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v15, v6, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[6:7], v[15:16] offset0:12 offset1:13
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v6, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v15, v5, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[15:16], v[4:5] offset0:12 offset1:13
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v14, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v14, v6, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[15:16], v[6:7] offset0:10 offset1:11
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v4
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v5, v14, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v14, v7, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v1
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v2
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[6:7], v[14:15] offset0:14 offset1:15
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v2, 0, 16
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[16:17], v[14:15] offset0:8 offset1:9
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v14, v2, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v3, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v8, v8, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v10, v9, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v12, v11, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v0, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v0, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v1, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[2:3], v[4:5] offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[6:7], v[12:13] offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[0:1], v[10:11] offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[16:17], v[8:9] offset1:1
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[2:3], v[5:6] offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[14:15], v[12:13] offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[0:1], v[10:11] offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v20, v[18:19], v[8:9] offset1:1
; GFX9-NO-DS128-NEXT: s_endpgm
;
; EG-LABEL: local_sextload_v16i16_to_v16i64:
@@ -7132,57 +7130,57 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i64(ptr addrspace(3) %out
; VI-DS128-NEXT: v_mov_b32_e32 v0, s1
; VI-DS128-NEXT: ds_read_b128 v[3:6], v0
; VI-DS128-NEXT: ds_read_b128 v[7:10], v0 offset:16
+; VI-DS128-NEXT: v_mov_b32_e32 v16, s0
; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
; VI-DS128-NEXT: v_bfe_i32 v0, v3, 0, 16
; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v8
-; VI-DS128-NEXT: v_bfe_i32 v11, v8, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v10
+; VI-DS128-NEXT: v_bfe_i32 v11, v10, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v13, v3, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; VI-DS128-NEXT: v_mov_b32_e32 v8, s0
-; VI-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:80
-; VI-DS128-NEXT: v_bfe_i32 v11, v7, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; VI-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v10
-; VI-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:64
-; VI-DS128-NEXT: v_bfe_i32 v11, v10, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v10, v9, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; VI-DS128-NEXT: ds_write_b128 v16, v[11:14] offset:112
+; VI-DS128-NEXT: v_bfe_i32 v12, v9, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; VI-DS128-NEXT: v_bfe_i32 v9, v8, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; VI-DS128-NEXT: ds_write_b128 v16, v[10:13] offset:96
+; VI-DS128-NEXT: v_bfe_i32 v11, v8, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; VI-DS128-NEXT: v_bfe_i32 v10, v4, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v8, v4, 0, 16
; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; VI-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:112
-; VI-DS128-NEXT: v_bfe_i32 v12, v4, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v9
-; VI-DS128-NEXT: v_bfe_i32 v14, v9, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
+; VI-DS128-NEXT: ds_write_b128 v16, v[9:12] offset:80
+; VI-DS128-NEXT: v_bfe_i32 v10, v4, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v7
+; VI-DS128-NEXT: v_bfe_i32 v12, v7, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v14, v4, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; VI-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v5
; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; VI-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:96
-; VI-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
+; VI-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:64
+; VI-DS128-NEXT: v_bfe_i32 v14, v4, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v14, v5, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v12, v5, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; VI-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; VI-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; VI-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; VI-DS128-NEXT: ds_write_b128 v8, v[4:7] offset:48
-; VI-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:32
-; VI-DS128-NEXT: ds_write_b128 v8, v[10:13] offset:16
-; VI-DS128-NEXT: ds_write_b128 v8, v[0:3]
+; VI-DS128-NEXT: ds_write_b128 v16, v[4:7] offset:48
+; VI-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:32
+; VI-DS128-NEXT: ds_write_b128 v16, v[8:11] offset:16
+; VI-DS128-NEXT: ds_write_b128 v16, v[0:3]
; VI-DS128-NEXT: s_endpgm
;
; GFX9-DS128-LABEL: local_sextload_v16i16_to_v16i64:
@@ -7192,57 +7190,57 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i64(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_mov_b32_e32 v0, s1
; GFX9-DS128-NEXT: ds_read_b128 v[3:6], v0
; GFX9-DS128-NEXT: ds_read_b128 v[7:10], v0 offset:16
+; GFX9-DS128-NEXT: v_mov_b32_e32 v16, s0
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
; GFX9-DS128-NEXT: v_bfe_i32 v0, v3, 0, 16
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v8
-; GFX9-DS128-NEXT: v_bfe_i32 v11, v8, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v10
+; GFX9-DS128-NEXT: v_bfe_i32 v11, v10, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v13, v3, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_mov_b32_e32 v8, s0
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:80
-; GFX9-DS128-NEXT: v_bfe_i32 v11, v7, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v10
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:64
-; GFX9-DS128-NEXT: v_bfe_i32 v11, v10, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v10, v9, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v9
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[11:14] offset:112
+; GFX9-DS128-NEXT: v_bfe_i32 v12, v9, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GFX9-DS128-NEXT: v_bfe_i32 v9, v8, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v8
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[10:13] offset:96
+; GFX9-DS128-NEXT: v_bfe_i32 v11, v8, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_bfe_i32 v10, v4, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v8, v4, 0, 16
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[11:14] offset:112
-; GFX9-DS128-NEXT: v_bfe_i32 v12, v4, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v9
-; GFX9-DS128-NEXT: v_bfe_i32 v14, v9, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[9:12] offset:80
+; GFX9-DS128-NEXT: v_bfe_i32 v10, v4, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v7
+; GFX9-DS128-NEXT: v_bfe_i32 v12, v7, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v4, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v5
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:96
-; GFX9-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:64
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v4, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v14, v5, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v12, v5, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[4:7] offset:48
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:32
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[10:13] offset:16
-; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3]
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[4:7] offset:48
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:32
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[8:11] offset:16
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[0:3]
; GFX9-DS128-NEXT: s_endpgm
%load = load <16 x i16>, ptr addrspace(3) %in
%ext = sext <16 x i16> %load to <16 x i64>
diff --git a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
index c45b584c5e90bb..0df1d73805bb67 100644
--- a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
+++ b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
@@ -183,22 +183,22 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:328
; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:332
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:328
-; ALIGNED-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:332
-; ALIGNED-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:320
-; ALIGNED-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:324
+; ALIGNED-NEXT: buffer_load_dword v39, off, s[0:3], s32 offset:332
+; ALIGNED-NEXT: buffer_load_dword v34, off, s[0:3], s32 offset:328
+; ALIGNED-NEXT: buffer_load_dword v33, off, s[0:3], s32 offset:324
+; ALIGNED-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:320
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v39 offset:36
-; ALIGNED-NEXT: flat_store_byte v[20:21], v39 offset:37
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v39 offset:40
+; ALIGNED-NEXT: flat_store_byte v[20:21], v39 offset:41
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v34 offset:40
-; ALIGNED-NEXT: flat_store_byte v[20:21], v34 offset:41
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v34 offset:36
+; ALIGNED-NEXT: flat_store_byte v[20:21], v34 offset:37
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v33 offset:28
-; ALIGNED-NEXT: flat_store_byte v[24:25], v33 offset:32
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v33 offset:32
+; ALIGNED-NEXT: flat_store_byte v[20:21], v33 offset:33
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v32 offset:32
-; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:33
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v32 offset:28
+; ALIGNED-NEXT: flat_store_byte v[24:25], v32 offset:32
; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:272
; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:276
; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:280
@@ -507,7 +507,7 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v32, 8, v32
; ALIGNED-NEXT: v_lshrrev_b32_e32 v102, 24, v67
; ALIGNED-NEXT: v_lshrrev_b32_e32 v67, 8, v67
-; ALIGNED-NEXT: flat_store_byte v[20:21], v101 offset:36
+; ALIGNED-NEXT: flat_store_byte v[20:21], v101 offset:32
; ALIGNED-NEXT: v_lshrrev_b32_e32 v101, 24, v54
; ALIGNED-NEXT: v_lshrrev_b32_e32 v54, 8, v54
; ALIGNED-NEXT: v_lshrrev_b32_e32 v103, 24, v66
@@ -523,25 +523,25 @@ define void @memcpy_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v81
; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 24, v97
; ALIGNED-NEXT: v_lshrrev_b32_e32 v97, 8, v97
-; ALIGNED-NEXT: flat_store_byte v[20:21], v98 offset:40
+; ALIGNED-NEXT: flat_store_byte v[20:21], v98 offset:44
; ALIGNED-NEXT: v_lshrrev_b32_e32 v98, 24, v87
; ALIGNED-NEXT: v_lshrrev_b32_e32 v87, 8, v87
-; ALIGNED-NEXT: flat_store_byte v[20:21], v39 offset:38
+; ALIGNED-NEXT: flat_store_byte v[20:21], v39 offset:42
; ALIGNED-NEXT: v_lshrrev_b32_e32 v39, 24, v83
; ALIGNED-NEXT: v_lshrrev_b32_e32 v83, 8, v83
-; ALIGNED-NEXT: flat_store_byte v[20:21], v99 offset:44
+; ALIGNED-NEXT: flat_store_byte v[20:21], v99 offset:40
; ALIGNED-NEXT: v_lshrrev_b32_e32 v99, 24, v68
; ALIGNED-NEXT: v_lshrrev_b32_e32 v68, 8, v68
-; ALIGNED-NEXT: flat_store_byte v[20:21], v34 offset:42
+; ALIGNED-NEXT: flat_store_byte v[20:21], v34 offset:38
; ALIGNED-NEXT: v_lshrrev_b32_e32 v34, 24, v96
; ALIGNED-NEXT: v_lshrrev_b32_e32 v96, 8, v96
-; ALIGNED-NEXT: flat_store_byte v[20:21], v100 offset:32
+; ALIGNED-NEXT: flat_store_byte v[20:21], v100 offset:36
; ALIGNED-NEXT: v_lshrrev_b32_e32 v100, 24, v84
; ALIGNED-NEXT: v_lshrrev_b32_e32 v84, 8, v84
-; ALIGNED-NEXT: flat_store_byte v[20:21], v33 offset:30
+; ALIGNED-NEXT: flat_store_byte v[20:21], v33 offset:34
; ALIGNED-NEXT: v_lshrrev_b32_e32 v33, 24, v69
; ALIGNED-NEXT: v_lshrrev_b32_e32 v69, 8, v69
-; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:34
+; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:30
; ALIGNED-NEXT: v_lshrrev_b32_e32 v32, 24, v85
; ALIGNED-NEXT: v_lshrrev_b32_e32 v85, 8, v85
; ALIGNED-NEXT: flat_store_byte v[20:21], v102 offset:28
@@ -1191,22 +1191,22 @@ define void @memcpy_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1)
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:296
; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:300
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:296
-; ALIGNED-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:300
-; ALIGNED-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:288
-; ALIGNED-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:300
+; ALIGNED-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:296
+; ALIGNED-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[20:21], v15, off offset:42
-; ALIGNED-NEXT: global_store_byte v[20:21], v15, off offset:40
+; ALIGNED-NEXT: global_store_byte_d16_hi v[20:21], v15, off offset:46
+; ALIGNED-NEXT: global_store_byte v[20:21], v15, off offset:44
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[20:21], v14, off offset:46
-; ALIGNED-NEXT: global_store_byte v[20:21], v14, off offset:44
+; ALIGNED-NEXT: global_store_byte_d16_hi v[20:21], v14, off offset:42
+; ALIGNED-NEXT: global_store_byte v[20:21], v14, off offset:40
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[20:21], v13, off offset:34
-; ALIGNED-NEXT: global_store_byte v[20:21], v13, off offset:32
+; ALIGNED-NEXT: global_store_byte_d16_hi v[20:21], v13, off offset:38
+; ALIGNED-NEXT: global_store_byte v[20:21], v13, off offset:36
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[20:21], v12, off offset:38
-; ALIGNED-NEXT: global_store_byte v[20:21], v12, off offset:36
+; ALIGNED-NEXT: global_store_byte_d16_hi v[20:21], v12, off offset:34
+; ALIGNED-NEXT: global_store_byte v[20:21], v12, off offset:32
; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:244
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248
@@ -1449,14 +1449,14 @@ define void @memcpy_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1)
; ALIGNED-NEXT: global_store_byte v[20:21], v17, off offset:53
; ALIGNED-NEXT: global_store_byte v[20:21], v82, off offset:51
; ALIGNED-NEXT: global_store_byte v[20:21], v16, off offset:49
-; ALIGNED-NEXT: global_store_byte v[20:21], v71, off offset:43
-; ALIGNED-NEXT: global_store_byte v[20:21], v15, off offset:41
-; ALIGNED-NEXT: global_store_byte v[20:21], v66, off offset:47
-; ALIGNED-NEXT: global_store_byte v[20:21], v14, off offset:45
-; ALIGNED-NEXT: global_store_byte v[20:21], v30, off offset:35
-; ALIGNED-NEXT: global_store_byte v[20:21], v13, off offset:33
-; ALIGNED-NEXT: global_store_byte v[20:21], v98, off offset:39
-; ALIGNED-NEXT: global_store_byte v[20:21], v12, off offset:37
+; ALIGNED-NEXT: global_store_byte v[20:21], v71, off offset:47
+; ALIGNED-NEXT: global_store_byte v[20:21], v15, off offset:45
+; ALIGNED-NEXT: global_store_byte v[20:21], v66, off offset:43
+; ALIGNED-NEXT: global_store_byte v[20:21], v14, off offset:41
+; ALIGNED-NEXT: global_store_byte v[20:21], v30, off offset:39
+; ALIGNED-NEXT: global_store_byte v[20:21], v13, off offset:37
+; ALIGNED-NEXT: global_store_byte v[20:21], v98, off offset:35
+; ALIGNED-NEXT: global_store_byte v[20:21], v12, off offset:33
; ALIGNED-NEXT: global_store_byte v[20:21], v70, off offset:31
; ALIGNED-NEXT: global_store_byte v[20:21], v11, off offset:29
; ALIGNED-NEXT: global_store_byte v[20:21], v83, off offset:27
@@ -1777,13 +1777,13 @@ define void @memcpy_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4)
; ALIGNED-NEXT: flat_store_byte v[86:87], v112 offset:198
; ALIGNED-NEXT: v_lshrrev_b32_e32 v112, 8, v55
; ALIGNED-NEXT: flat_store_byte v[86:87], v80 offset:204
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v80, 24, v52
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v80, 24, v53
; ALIGNED-NEXT: flat_store_byte v[86:87], v81 offset:202
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v52
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v53
; ALIGNED-NEXT: flat_store_byte v[86:87], v82 offset:196
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v82, 24, v53
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v82, 24, v52
; ALIGNED-NEXT: flat_store_byte v[86:87], v83 offset:194
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v83, 8, v53
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v83, 8, v52
; ALIGNED-NEXT: flat_store_byte v[86:87], v114 offset:192
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 24, v50
@@ -1815,18 +1815,18 @@ define void @memcpy_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4)
; ALIGNED-NEXT: flat_store_byte v[86:87], v103 offset:178
; ALIGNED-NEXT: flat_store_byte v[86:87], v68 offset:176
; ALIGNED-NEXT: flat_store_byte v[86:87], v69 offset:174
-; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:252
; ALIGNED-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:252
; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:244
; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v54 offset:164
-; ALIGNED-NEXT: flat_store_byte v[86:87], v54 offset:165
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v55 offset:168
; ALIGNED-NEXT: flat_store_byte v[86:87], v55 offset:169
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v52 offset:156
-; ALIGNED-NEXT: flat_store_byte v[86:87], v52 offset:157
+; ALIGNED-NEXT: flat_store_byte v[86:87], v54 offset:165
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v53 offset:160
; ALIGNED-NEXT: flat_store_byte v[86:87], v53 offset:161
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v52 offset:156
+; ALIGNED-NEXT: flat_store_byte v[86:87], v52 offset:157
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v52, 24, v36
; ALIGNED-NEXT: v_lshrrev_b32_e32 v100, 24, v38
@@ -1847,12 +1847,12 @@ define void @memcpy_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v113, 24, v33
; ALIGNED-NEXT: flat_store_byte v[86:87], v112 offset:170
; ALIGNED-NEXT: v_lshrrev_b32_e32 v112, 8, v33
-; ALIGNED-NEXT: flat_store_byte v[86:87], v80 offset:160
+; ALIGNED-NEXT: flat_store_byte v[86:87], v80 offset:164
; ALIGNED-NEXT: v_lshrrev_b32_e32 v80, 24, v32
-; ALIGNED-NEXT: flat_store_byte v[86:87], v81 offset:158
+; ALIGNED-NEXT: flat_store_byte v[86:87], v81 offset:162
; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v32
-; ALIGNED-NEXT: flat_store_byte v[86:87], v82 offset:164
-; ALIGNED-NEXT: flat_store_byte v[86:87], v83 offset:162
+; ALIGNED-NEXT: flat_store_byte v[86:87], v82 offset:160
+; ALIGNED-NEXT: flat_store_byte v[86:87], v83 offset:158
; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:200
; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:204
; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:196
@@ -3786,37 +3786,37 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:29
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:30
; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:31
-; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:32
; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:33
-; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:34
-; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:35
-; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:36
-; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:37
-; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:38
-; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:39
-; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:40
+; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:39
+; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:40
; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:41
; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:42
-; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:43
-; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:44
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:45
-; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:46
-; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:44
+; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:47
; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:48
; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:49
; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:43
; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:51
; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:52
-; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:53
-; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:54
-; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:55
+; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:55
; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:56
; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:57
; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:58
-; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:60
-; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:61
-; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:62
-; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:60
+; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:63
; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:64
; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:65
; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:66
@@ -3864,47 +3864,45 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: v_lshl_or_b32 v5, v8, 8, v6
; ALIGNED-NEXT: v_lshl_or_b32 v6, v12, 8, v11
; ALIGNED-NEXT: s_waitcnt vmcnt(45)
-; ALIGNED-NEXT: v_lshl_or_b32 v7, v15, 8, v14
+; ALIGNED-NEXT: v_lshl_or_b32 v7, v15, 8, v13
; ALIGNED-NEXT: s_waitcnt vmcnt(43)
-; ALIGNED-NEXT: v_lshl_or_b32 v8, v20, 8, v19
+; ALIGNED-NEXT: v_lshl_or_b32 v8, v17, 8, v16
; ALIGNED-NEXT: s_waitcnt vmcnt(41)
-; ALIGNED-NEXT: v_lshl_or_b32 v9, v16, 8, v13
+; ALIGNED-NEXT: v_lshl_or_b32 v9, v18, 8, v14
; ALIGNED-NEXT: s_waitcnt vmcnt(39)
-; ALIGNED-NEXT: v_lshl_or_b32 v10, v18, 8, v17
-; ALIGNED-NEXT: s_waitcnt vmcnt(37)
-; ALIGNED-NEXT: v_lshl_or_b32 v11, v27, 8, v22
+; ALIGNED-NEXT: v_lshl_or_b32 v10, v20, 8, v19
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: s_waitcnt vmcnt(35)
-; ALIGNED-NEXT: v_lshl_or_b32 v12, v29, 8, v28
; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v6, 16, v5
+; ALIGNED-NEXT: s_waitcnt vmcnt(34)
+; ALIGNED-NEXT: v_lshl_or_b32 v11, v23, 8, v21
; ALIGNED-NEXT: v_lshl_or_b32 v4, v8, 16, v7
+; ALIGNED-NEXT: s_waitcnt vmcnt(32)
+; ALIGNED-NEXT: v_lshl_or_b32 v12, v25, 8, v24
; ALIGNED-NEXT: v_lshl_or_b32 v5, v10, 16, v9
-; ALIGNED-NEXT: v_lshl_or_b32 v6, v12, 16, v11
-; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:496 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(33)
-; ALIGNED-NEXT: v_lshl_or_b32 v13, v24, 8, v23
-; ALIGNED-NEXT: s_waitcnt vmcnt(31)
-; ALIGNED-NEXT: v_lshl_or_b32 v14, v26, 8, v25
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(29)
+; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v13, v27, 8, v26
+; ALIGNED-NEXT: v_lshl_or_b32 v6, v12, 16, v11
+; ALIGNED-NEXT: s_waitcnt vmcnt(28)
+; ALIGNED-NEXT: v_lshl_or_b32 v14, v29, 8, v28
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v31, 8, v30
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:524 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(27)
; ALIGNED-NEXT: v_lshl_or_b32 v1, v34, 8, v33
; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:528 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(25)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v37, 8, v32
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v35, 8, v32
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:576 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(23)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v36, 8, v35
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v37, 8, v36
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:580 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(18)
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v49, 8, v21
+; ALIGNED-NEXT: v_lshl_or_b32 v5, v39, 8, v22
; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:592 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(16)
-; ALIGNED-NEXT: v_lshl_or_b32 v6, v48, 8, v39
+; ALIGNED-NEXT: v_lshl_or_b32 v6, v49, 8, v48
; ALIGNED-NEXT: v_lshl_or_b32 v7, v14, 16, v13
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 16, v3
@@ -3912,7 +3910,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v6, 16, v5
; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:612 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v7, v50, 8, v38
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(13)
; ALIGNED-NEXT: v_lshl_or_b32 v0, v52, 8, v51
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:656 ; 4-byte Folded Spill
@@ -3931,43 +3929,43 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: s_waitcnt vmcnt(14)
; ALIGNED-NEXT: v_lshl_or_b32 v0, v66, 8, v65
; ALIGNED-NEXT: buffer_load_ubyte v3, v2, s[0:3], 0 offen offset:81
-; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:508 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:700 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(13)
; ALIGNED-NEXT: v_lshl_or_b32 v1, v68, 8, v67
-; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:500 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:536 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:532 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:79
-; ALIGNED-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:548 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:512 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
; ALIGNED-NEXT: v_lshl_or_b32 v0, v82, 8, v81
-; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:540 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:552 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:556 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:516 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:520 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:544 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:560 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:568 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:564 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:572 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:584 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:600 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:588 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:596 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:616 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:604 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:608 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:624 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:628 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:632 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:636 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:620 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:640 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:648 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:652 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:644 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:664 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:672 ; 4-byte Folded Spill
@@ -4219,36 +4217,36 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:144
; ALIGNED-NEXT: v_lshl_or_b32 v0, v0, 8, v5
; ALIGNED-NEXT: s_clause 0x3e
-; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:165
-; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:166
; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:167
; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:168
; ALIGNED-NEXT: buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:169
-; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:170
-; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:171
-; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:170
+; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:172
; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:173
; ALIGNED-NEXT: buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:174
; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:175
-; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:176
-; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:177
-; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT: buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:178
; ALIGNED-NEXT: buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:179
-; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:180
; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:181
; ALIGNED-NEXT: buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:182
; ALIGNED-NEXT: buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:183
; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:184
; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:185
-; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:186
; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:187
-; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:188
; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:189
-; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:190
-; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:191
; ALIGNED-NEXT: buffer_load_ubyte v116, v2, s[0:3], 0 offen offset:192
; ALIGNED-NEXT: buffer_load_ubyte v114, v2, s[0:3], 0 offen offset:193
-; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT: buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:194
; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:195
; ALIGNED-NEXT: buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:196
; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:197
@@ -4257,24 +4255,24 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:200
; ALIGNED-NEXT: buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:201
; ALIGNED-NEXT: buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:202
-; ALIGNED-NEXT: buffer_load_ubyte v96, v2, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:203
; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:204
; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:205
; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:206
; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:207
; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:208
; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:209
-; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:210
; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:211
-; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:212
-; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:213
-; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:214
-; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:215
-; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:216
-; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:217
-; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:218
-; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:219
-; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:220
; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:221
; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:222
; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:223
@@ -4307,15 +4305,15 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:4
; ALIGNED-NEXT: buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:5
; ALIGNED-NEXT: buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:6
-; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:7
-; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:12
-; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:13
-; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:14
; ALIGNED-NEXT: buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:15
; ALIGNED-NEXT: buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:11
; ALIGNED-NEXT: buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:8
; ALIGNED-NEXT: buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:9
-; ALIGNED-NEXT: buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:10
+; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:10
; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:145
; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:146
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:147
@@ -4423,13 +4421,13 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v83, 8, v85
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1128 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v7, 8, v6
-; ALIGNED-NEXT: v_lshl_or_b32 v61, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v75, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v97, 8, v98
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v96, 8, v87
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v86, 8, v87
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v13, 8, v12
-; ALIGNED-NEXT: v_lshl_or_b32 v59, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v53, 8, v68
+; ALIGNED-NEXT: v_lshl_or_b32 v60, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v71, 8, v81
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1144 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v11, 8, v10
; ALIGNED-NEXT: s_clause 0x7
@@ -4441,62 +4439,62 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:250
; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:251
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:252
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v52, 8, v64
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v70, 8, v69
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v108, 8, v109
-; ALIGNED-NEXT: v_lshl_or_b32 v117, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v65, 8, v67
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v54, 8, v55
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v108, 8, v110
+; ALIGNED-NEXT: v_lshl_or_b32 v43, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v67, 8, v80
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v65, 8, v66
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1164 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v111, 8, v5
-; ALIGNED-NEXT: v_lshl_or_b32 v103, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v49, 8, v66
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v109, 8, v5
+; ALIGNED-NEXT: v_lshl_or_b32 v113, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v49, 8, v51
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v79, 8, v88
; ALIGNED-NEXT: v_lshl_or_b32 v4, v48, 8, v50
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1180 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v89, 8, v93
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v89, 8, v91
; ALIGNED-NEXT: v_lshl_or_b32 v99, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v71, 8, v81
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v70, 8, v80
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v54, 8, v64
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v52, 8, v53
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v104, 8, v95
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v95, 8, v94
+; ALIGNED-NEXT: v_lshl_or_b32 v96, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v36, 8, v38
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v105, 8, v107
; ALIGNED-NEXT: v_lshl_or_b32 v4, v34, 8, v35
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v62, 8, v63
-; ALIGNED-NEXT: v_lshl_or_b32 v69, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v62, 8, v61
+; ALIGNED-NEXT: v_lshl_or_b32 v68, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v31, 8, v37
; ALIGNED-NEXT: v_lshl_or_b32 v4, v30, 8, v32
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1196 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v72, 8, v75
-; ALIGNED-NEXT: v_lshl_or_b32 v51, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v63, 8, v74
+; ALIGNED-NEXT: v_lshl_or_b32 v55, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v23, 8, v25
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
; ALIGNED-NEXT: v_lshl_or_b32 v1, v46, 8, v58
; ALIGNED-NEXT: v_lshl_or_b32 v4, v22, 8, v24
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1204 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v47, 8, v73
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v47, 8, v72
; ALIGNED-NEXT: v_lshl_or_b32 v39, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v28, 8, v29
; ALIGNED-NEXT: v_lshl_or_b32 v4, v26, 8, v27
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v118, 8, v40
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v117, 8, v118
; ALIGNED-NEXT: v_lshl_or_b32 v33, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v19, 8, v21
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1208 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v0, v119, 8, v41
+; ALIGNED-NEXT: v_lshl_or_b32 v0, v119, 8, v40
; ALIGNED-NEXT: v_lshl_or_b32 v4, v18, 8, v17
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v42, 8, v43
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v42, 8, v41
; ALIGNED-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1212 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v44, 8, v45
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
-; ALIGNED-NEXT: v_lshl_or_b32 v1, v112, 8, v113
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v112, 8, v103
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1216 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v114, 8, v116
; ALIGNED-NEXT: v_lshl_or_b32 v0, v1, 16, v0
@@ -4525,23 +4523,23 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: v_lshl_or_b32 v56, v124, 8, v126
; ALIGNED-NEXT: v_lshl_or_b32 v0, v57, 16, v56
; ALIGNED-NEXT: v_lshl_or_b32 v56, v123, 8, v125
-; ALIGNED-NEXT: v_lshl_or_b32 v57, v110, 8, v122
+; ALIGNED-NEXT: v_lshl_or_b32 v57, v111, 8, v122
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1172 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v57, 16, v56
-; ALIGNED-NEXT: v_lshl_or_b32 v56, v91, 8, v94
-; ALIGNED-NEXT: v_lshl_or_b32 v57, v90, 8, v92
+; ALIGNED-NEXT: v_lshl_or_b32 v56, v92, 8, v104
+; ALIGNED-NEXT: v_lshl_or_b32 v57, v90, 8, v93
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1176 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v57, 16, v56
; ALIGNED-NEXT: v_lshl_or_b32 v56, v76, 8, v78
-; ALIGNED-NEXT: v_lshl_or_b32 v57, v77, 8, v74
+; ALIGNED-NEXT: v_lshl_or_b32 v57, v77, 8, v73
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1184 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v57, 16, v56
; ALIGNED-NEXT: s_clause 0x2
; ALIGNED-NEXT: buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:18
-; ALIGNED-NEXT: buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:16
+; ALIGNED-NEXT: buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:16
; ALIGNED-NEXT: buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:17
; ALIGNED-NEXT: v_add_nc_u32_e32 v2, 0x100, v2
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:232
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:236
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
@@ -4549,7 +4547,7 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
; ALIGNED-NEXT: v_lshl_or_b32 v0, v127, 8, v56
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v127, v57, 8, v60
+; ALIGNED-NEXT: v_lshl_or_b32 v127, v57, 8, v59
; ALIGNED-NEXT: v_lshl_or_b32 v127, v0, 16, v127
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1228 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
@@ -4579,8 +4577,8 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v21 offset:237
; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:248
; ALIGNED-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:252
-; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:244
-; ALIGNED-NEXT: buffer_store_dword v69, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT: buffer_store_dword v68, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: flat_store_byte v[5:6], v27 offset:231
; ALIGNED-NEXT: flat_store_byte v[5:6], v26 offset:232
; ALIGNED-NEXT: flat_store_byte v[5:6], v28 offset:230
@@ -4597,28 +4595,28 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v32 offset:227
; ALIGNED-NEXT: flat_store_byte v[5:6], v37 offset:225
; ALIGNED-NEXT: flat_store_byte v[5:6], v38 offset:221
-; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:200
; ALIGNED-NEXT: buffer_store_dword v99, off, s[0:3], s32 offset:204
-; ALIGNED-NEXT: buffer_store_dword v103, off, s[0:3], s32 offset:200
-; ALIGNED-NEXT: buffer_store_dword v117, off, s[0:3], s32 offset:196
-; ALIGNED-NEXT: flat_store_byte v[5:6], v53 offset:210
-; ALIGNED-NEXT: flat_store_byte v[5:6], v52 offset:212
-; ALIGNED-NEXT: flat_store_byte v[5:6], v71 offset:206
-; ALIGNED-NEXT: flat_store_byte v[5:6], v70 offset:208
-; ALIGNED-NEXT: flat_store_byte v[5:6], v80 offset:207
-; ALIGNED-NEXT: flat_store_byte v[5:6], v64 offset:211
-; ALIGNED-NEXT: flat_store_byte v[5:6], v68 offset:209
-; ALIGNED-NEXT: flat_store_byte v[5:6], v55 offset:215
-; ALIGNED-NEXT: flat_store_byte v[5:6], v54 offset:216
-; ALIGNED-NEXT: flat_store_byte v[5:6], v65 offset:214
+; ALIGNED-NEXT: buffer_store_dword v113, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT: buffer_store_dword v43, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT: flat_store_byte v[5:6], v53 offset:215
+; ALIGNED-NEXT: flat_store_byte v[5:6], v52 offset:216
+; ALIGNED-NEXT: flat_store_byte v[5:6], v54 offset:214
; ALIGNED-NEXT: flat_store_byte v[5:6], v48 offset:220
; ALIGNED-NEXT: flat_store_byte v[5:6], v49 offset:218
; ALIGNED-NEXT: flat_store_byte v[5:6], v50 offset:219
-; ALIGNED-NEXT: flat_store_byte v[5:6], v66 offset:217
-; ALIGNED-NEXT: flat_store_byte v[5:6], v67 offset:213
+; ALIGNED-NEXT: flat_store_byte v[5:6], v51 offset:217
+; ALIGNED-NEXT: flat_store_byte v[5:6], v64 offset:213
+; ALIGNED-NEXT: flat_store_byte v[5:6], v69 offset:207
+; ALIGNED-NEXT: flat_store_byte v[5:6], v70 offset:208
+; ALIGNED-NEXT: flat_store_byte v[5:6], v71 offset:206
+; ALIGNED-NEXT: flat_store_byte v[5:6], v65 offset:212
+; ALIGNED-NEXT: flat_store_byte v[5:6], v67 offset:210
+; ALIGNED-NEXT: flat_store_byte v[5:6], v66 offset:211
+; ALIGNED-NEXT: flat_store_byte v[5:6], v80 offset:209
; ALIGNED-NEXT: flat_store_byte v[5:6], v81 offset:205
-; ALIGNED-NEXT: buffer_store_dword v59, off, s[0:3], s32 offset:216
-; ALIGNED-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT: buffer_store_dword v75, off, s[0:3], s32 offset:220
; ALIGNED-NEXT: buffer_store_dword v106, off, s[0:3], s32 offset:212
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1220 ; 4-byte Folded Reload
; ALIGNED-NEXT: v_cmp_gt_u64_e64 vcc_lo, 0x800, s[4:5]
@@ -4626,14 +4624,14 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:208
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1216 ; 4-byte Folded Reload
; ALIGNED-NEXT: flat_store_byte v[5:6], v87 offset:199
-; ALIGNED-NEXT: flat_store_byte v[5:6], v96 offset:200
+; ALIGNED-NEXT: flat_store_byte v[5:6], v86 offset:200
; ALIGNED-NEXT: flat_store_byte v[5:6], v97 offset:198
; ALIGNED-NEXT: flat_store_byte v[5:6], v82 offset:204
; ALIGNED-NEXT: flat_store_byte v[5:6], v83 offset:202
; ALIGNED-NEXT: flat_store_byte v[5:6], v84 offset:203
; ALIGNED-NEXT: flat_store_byte v[5:6], v85 offset:201
; ALIGNED-NEXT: flat_store_byte v[5:6], v98 offset:197
-; ALIGNED-NEXT: flat_store_byte v[5:6], v113 offset:191
+; ALIGNED-NEXT: flat_store_byte v[5:6], v103 offset:191
; ALIGNED-NEXT: flat_store_byte v[5:6], v112 offset:192
; ALIGNED-NEXT: flat_store_byte v[5:6], v114 offset:190
; ALIGNED-NEXT: flat_store_byte v[5:6], v100 offset:196
@@ -4653,25 +4651,25 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1196 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[5:6], v43 offset:183
+; ALIGNED-NEXT: flat_store_byte v[5:6], v41 offset:183
; ALIGNED-NEXT: flat_store_byte v[5:6], v42 offset:184
; ALIGNED-NEXT: flat_store_byte v[5:6], v44 offset:182
-; ALIGNED-NEXT: flat_store_byte v[5:6], v118 offset:188
+; ALIGNED-NEXT: flat_store_byte v[5:6], v117 offset:188
; ALIGNED-NEXT: flat_store_byte v[5:6], v119 offset:186
-; ALIGNED-NEXT: flat_store_byte v[5:6], v40 offset:187
-; ALIGNED-NEXT: flat_store_byte v[5:6], v41 offset:185
+; ALIGNED-NEXT: flat_store_byte v[5:6], v118 offset:187
+; ALIGNED-NEXT: flat_store_byte v[5:6], v40 offset:185
; ALIGNED-NEXT: flat_store_byte v[5:6], v45 offset:181
-; ALIGNED-NEXT: flat_store_byte v[5:6], v63 offset:175
+; ALIGNED-NEXT: flat_store_byte v[5:6], v61 offset:175
; ALIGNED-NEXT: flat_store_byte v[5:6], v62 offset:176
-; ALIGNED-NEXT: flat_store_byte v[5:6], v72 offset:174
+; ALIGNED-NEXT: flat_store_byte v[5:6], v63 offset:174
; ALIGNED-NEXT: flat_store_byte v[5:6], v46 offset:180
; ALIGNED-NEXT: flat_store_byte v[5:6], v47 offset:178
; ALIGNED-NEXT: flat_store_byte v[5:6], v58 offset:179
-; ALIGNED-NEXT: flat_store_byte v[5:6], v73 offset:177
-; ALIGNED-NEXT: flat_store_byte v[5:6], v75 offset:173
+; ALIGNED-NEXT: flat_store_byte v[5:6], v72 offset:177
+; ALIGNED-NEXT: flat_store_byte v[5:6], v74 offset:173
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:316
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1180 ; 4-byte Folded Reload
@@ -4681,13 +4679,13 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1160 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[5:6], v95 offset:167
-; ALIGNED-NEXT: flat_store_byte v[5:6], v104 offset:168
+; ALIGNED-NEXT: flat_store_byte v[5:6], v94 offset:167
+; ALIGNED-NEXT: flat_store_byte v[5:6], v95 offset:168
; ALIGNED-NEXT: flat_store_byte v[5:6], v105 offset:166
; ALIGNED-NEXT: flat_store_byte v[5:6], v79 offset:172
; ALIGNED-NEXT: flat_store_byte v[5:6], v89 offset:170
; ALIGNED-NEXT: flat_store_byte v[5:6], v88 offset:171
-; ALIGNED-NEXT: flat_store_byte v[5:6], v93 offset:169
+; ALIGNED-NEXT: flat_store_byte v[5:6], v91 offset:169
; ALIGNED-NEXT: flat_store_byte v[5:6], v107 offset:165
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:159
@@ -4698,8 +4696,8 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:158
; ALIGNED-NEXT: flat_store_byte v[5:6], v108 offset:164
-; ALIGNED-NEXT: flat_store_byte v[5:6], v111 offset:162
-; ALIGNED-NEXT: flat_store_byte v[5:6], v109 offset:163
+; ALIGNED-NEXT: flat_store_byte v[5:6], v109 offset:162
+; ALIGNED-NEXT: flat_store_byte v[5:6], v110 offset:163
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1168 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:161
@@ -5075,13 +5073,10 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:656 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:58
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:648 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:652 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:55
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:644 ; 4-byte Folded Reload
@@ -5090,11 +5085,14 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:640 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:54
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:636 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:60
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:628 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:58
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:632 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:59
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:624 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
@@ -5102,21 +5100,21 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:620 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:53
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:50
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:47
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:600 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:48
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:584 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:46
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:608 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:616 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:52
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:596 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:50
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:604 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:51
@@ -5128,10 +5126,10 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:45
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:612 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:592 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:580 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436
@@ -5140,59 +5138,59 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:432
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:568 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:40
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:39
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:564 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:39
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:40
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:560 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:38
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:37
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:556 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:44
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:548 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:42
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:552 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:43
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:42
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:540 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:41
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:544 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:37
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:536 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:32
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:31
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:532 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:31
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:32
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:30
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:520 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:36
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:512 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:34
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:516 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:35
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:500 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:34
+; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:33
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:496 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:33
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:528 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:392
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:524 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:396
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:508 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:504 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:388
; ALIGNED-NEXT: buffer_store_dword v127, off, s[0:3], s32 offset:384
@@ -5237,8 +5235,8 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:448 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[5:6], v0 offset:17
-; ALIGNED-NEXT: flat_store_byte v[3:4], v60 offset:16
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1192 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v59 offset:16
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1188 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1184 ; 4-byte Folded Reload
@@ -5250,18 +5248,18 @@ define void @memcpy_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5)
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1172 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400
-; ALIGNED-NEXT: flat_store_byte v[5:6], v74 offset:7
+; ALIGNED-NEXT: flat_store_byte v[5:6], v73 offset:7
; ALIGNED-NEXT: flat_store_byte v[5:6], v77 offset:8
-; ALIGNED-NEXT: flat_store_byte v[5:6], v91 offset:10
; ALIGNED-NEXT: flat_store_byte v[5:6], v76 offset:6
; ALIGNED-NEXT: flat_store_byte v[5:6], v90 offset:12
-; ALIGNED-NEXT: flat_store_byte v[5:6], v92 offset:11
-; ALIGNED-NEXT: flat_store_byte v[5:6], v94 offset:9
+; ALIGNED-NEXT: flat_store_byte v[5:6], v92 offset:10
+; ALIGNED-NEXT: flat_store_byte v[5:6], v93 offset:11
+; ALIGNED-NEXT: flat_store_byte v[5:6], v104 offset:9
; ALIGNED-NEXT: flat_store_byte v[3:4], v78 offset:8
; ALIGNED-NEXT: flat_store_byte v[3:4], v121 offset:2
; ALIGNED-NEXT: flat_store_byte v[5:6], v120
; ALIGNED-NEXT: flat_store_byte v[3:4], v124 offset:1
-; ALIGNED-NEXT: flat_store_byte v[5:6], v110 offset:4
+; ALIGNED-NEXT: flat_store_byte v[5:6], v111 offset:4
; ALIGNED-NEXT: flat_store_byte v[5:6], v123 offset:2
; ALIGNED-NEXT: flat_store_byte v[5:6], v122 offset:3
; ALIGNED-NEXT: flat_store_byte v[3:4], v125 offset:4
@@ -5631,22 +5629,22 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:296
; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:300
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:296
-; ALIGNED-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:300
-; ALIGNED-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:288
-; ALIGNED-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT: buffer_load_dword v27, off, s[0:3], s32 offset:300
+; ALIGNED-NEXT: buffer_load_dword v26, off, s[0:3], s32 offset:296
+; ALIGNED-NEXT: buffer_load_dword v25, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT: buffer_load_dword v24, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v27 offset:36
-; ALIGNED-NEXT: flat_store_byte v[20:21], v27 offset:37
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v27 offset:40
+; ALIGNED-NEXT: flat_store_byte v[20:21], v27 offset:41
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v26 offset:40
-; ALIGNED-NEXT: flat_store_byte v[20:21], v26 offset:41
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v26 offset:36
+; ALIGNED-NEXT: flat_store_byte v[20:21], v26 offset:37
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v25 offset:28
-; ALIGNED-NEXT: flat_store_byte v[0:1], v25 offset:32
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v25 offset:32
+; ALIGNED-NEXT: flat_store_byte v[20:21], v25 offset:33
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v24 offset:32
-; ALIGNED-NEXT: flat_store_byte v[20:21], v24 offset:33
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v24 offset:28
+; ALIGNED-NEXT: flat_store_byte v[0:1], v24 offset:32
; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:244
; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:248
@@ -5958,12 +5956,12 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: v_lshrrev_b32_e32 v98, 24, v25
; ALIGNED-NEXT: v_lshrrev_b32_e32 v25, 8, v25
; ALIGNED-NEXT: v_lshrrev_b32_e32 v100, 24, v31
-; ALIGNED-NEXT: flat_store_byte v[20:21], v99 offset:36
+; ALIGNED-NEXT: flat_store_byte v[20:21], v99 offset:32
; ALIGNED-NEXT: v_lshrrev_b32_e32 v99, 24, v49
; ALIGNED-NEXT: v_lshrrev_b32_e32 v31, 8, v31
; ALIGNED-NEXT: v_lshrrev_b32_e32 v49, 8, v49
; ALIGNED-NEXT: v_lshrrev_b32_e32 v101, 24, v30
-; ALIGNED-NEXT: flat_store_byte v[20:21], v24 offset:34
+; ALIGNED-NEXT: flat_store_byte v[20:21], v24 offset:30
; ALIGNED-NEXT: v_lshrrev_b32_e32 v24, 24, v48
; ALIGNED-NEXT: v_lshrrev_b32_e32 v48, 8, v48
; ALIGNED-NEXT: v_lshrrev_b32_e32 v30, 8, v30
@@ -5971,22 +5969,22 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: v_lshrrev_b32_e32 v29, 8, v29
; ALIGNED-NEXT: v_lshrrev_b32_e32 v103, 24, v28
; ALIGNED-NEXT: v_lshrrev_b32_e32 v28, 8, v28
-; ALIGNED-NEXT: flat_store_byte v[20:21], v96 offset:40
+; ALIGNED-NEXT: flat_store_byte v[20:21], v96 offset:44
; ALIGNED-NEXT: v_lshrrev_b32_e32 v96, 24, v39
; ALIGNED-NEXT: v_lshrrev_b32_e32 v39, 8, v39
-; ALIGNED-NEXT: flat_store_byte v[20:21], v27 offset:38
+; ALIGNED-NEXT: flat_store_byte v[20:21], v27 offset:42
; ALIGNED-NEXT: v_lshrrev_b32_e32 v27, 24, v38
; ALIGNED-NEXT: v_lshrrev_b32_e32 v38, 8, v38
-; ALIGNED-NEXT: flat_store_byte v[20:21], v97 offset:44
+; ALIGNED-NEXT: flat_store_byte v[20:21], v97 offset:40
; ALIGNED-NEXT: v_lshrrev_b32_e32 v97, 24, v37
; ALIGNED-NEXT: v_lshrrev_b32_e32 v37, 8, v37
-; ALIGNED-NEXT: flat_store_byte v[20:21], v26 offset:42
+; ALIGNED-NEXT: flat_store_byte v[20:21], v26 offset:38
; ALIGNED-NEXT: v_lshrrev_b32_e32 v26, 24, v36
; ALIGNED-NEXT: v_lshrrev_b32_e32 v36, 8, v36
-; ALIGNED-NEXT: flat_store_byte v[20:21], v98 offset:32
+; ALIGNED-NEXT: flat_store_byte v[20:21], v98 offset:36
; ALIGNED-NEXT: v_lshrrev_b32_e32 v98, 24, v51
; ALIGNED-NEXT: v_lshrrev_b32_e32 v51, 8, v51
-; ALIGNED-NEXT: flat_store_byte v[20:21], v25 offset:30
+; ALIGNED-NEXT: flat_store_byte v[20:21], v25 offset:34
; ALIGNED-NEXT: v_lshrrev_b32_e32 v25, 24, v50
; ALIGNED-NEXT: v_lshrrev_b32_e32 v50, 8, v50
; ALIGNED-NEXT: flat_store_byte v[20:21], v100 offset:28
@@ -6270,22 +6268,22 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:552
; ALIGNED-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:556
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:552
-; ALIGNED-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:556
-; ALIGNED-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:544
-; ALIGNED-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:548
+; ALIGNED-NEXT: buffer_load_dword v37, off, s[0:3], s32 offset:556
+; ALIGNED-NEXT: buffer_load_dword v32, off, s[0:3], s32 offset:552
+; ALIGNED-NEXT: buffer_load_dword v31, off, s[0:3], s32 offset:548
+; ALIGNED-NEXT: buffer_load_dword v30, off, s[0:3], s32 offset:544
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v37 offset:36
-; ALIGNED-NEXT: flat_store_byte v[20:21], v37 offset:37
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v37 offset:40
+; ALIGNED-NEXT: flat_store_byte v[20:21], v37 offset:41
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v32 offset:40
-; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:41
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v32 offset:36
+; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:37
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v31 offset:28
-; ALIGNED-NEXT: flat_store_byte v[16:17], v31 offset:32
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v31 offset:32
+; ALIGNED-NEXT: flat_store_byte v[20:21], v31 offset:33
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v30 offset:32
-; ALIGNED-NEXT: flat_store_byte v[20:21], v30 offset:33
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[22:23], v30 offset:28
+; ALIGNED-NEXT: flat_store_byte v[16:17], v30 offset:32
; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:496
; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:500
; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:504
@@ -6597,12 +6595,12 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: v_lshrrev_b32_e32 v98, 24, v31
; ALIGNED-NEXT: v_lshrrev_b32_e32 v31, 8, v31
; ALIGNED-NEXT: v_lshrrev_b32_e32 v100, 24, v65
-; ALIGNED-NEXT: flat_store_byte v[20:21], v99 offset:36
+; ALIGNED-NEXT: flat_store_byte v[20:21], v99 offset:32
; ALIGNED-NEXT: v_lshrrev_b32_e32 v99, 24, v67
; ALIGNED-NEXT: v_lshrrev_b32_e32 v65, 8, v65
; ALIGNED-NEXT: v_lshrrev_b32_e32 v67, 8, v67
; ALIGNED-NEXT: v_lshrrev_b32_e32 v101, 24, v64
-; ALIGNED-NEXT: flat_store_byte v[20:21], v30 offset:34
+; ALIGNED-NEXT: flat_store_byte v[20:21], v30 offset:30
; ALIGNED-NEXT: v_lshrrev_b32_e32 v30, 24, v52
; ALIGNED-NEXT: v_lshrrev_b32_e32 v52, 8, v52
; ALIGNED-NEXT: v_lshrrev_b32_e32 v64, 8, v64
@@ -6610,22 +6608,22 @@ define void @memmove_p0_p0_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(0
; ALIGNED-NEXT: v_lshrrev_b32_e32 v51, 8, v51
; ALIGNED-NEXT: v_lshrrev_b32_e32 v103, 24, v50
; ALIGNED-NEXT: v_lshrrev_b32_e32 v50, 8, v50
-; ALIGNED-NEXT: flat_store_byte v[20:21], v96 offset:40
+; ALIGNED-NEXT: flat_store_byte v[20:21], v96 offset:44
; ALIGNED-NEXT: v_lshrrev_b32_e32 v96, 24, v87
; ALIGNED-NEXT: v_lshrrev_b32_e32 v87, 8, v87
-; ALIGNED-NEXT: flat_store_byte v[20:21], v37 offset:38
+; ALIGNED-NEXT: flat_store_byte v[20:21], v37 offset:42
; ALIGNED-NEXT: v_lshrrev_b32_e32 v37, 24, v85
; ALIGNED-NEXT: v_lshrrev_b32_e32 v85, 8, v85
-; ALIGNED-NEXT: flat_store_byte v[20:21], v97 offset:44
+; ALIGNED-NEXT: flat_store_byte v[20:21], v97 offset:40
; ALIGNED-NEXT: v_lshrrev_b32_e32 v97, 24, v71
; ALIGNED-NEXT: v_lshrrev_b32_e32 v71, 8, v71
-; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:42
+; ALIGNED-NEXT: flat_store_byte v[20:21], v32 offset:38
; ALIGNED-NEXT: v_lshrrev_b32_e32 v32, 24, v66
; ALIGNED-NEXT: v_lshrrev_b32_e32 v66, 8, v66
-; ALIGNED-NEXT: flat_store_byte v[20:21], v98 offset:32
+; ALIGNED-NEXT: flat_store_byte v[20:21], v98 offset:36
; ALIGNED-NEXT: v_lshrrev_b32_e32 v98, 24, v86
; ALIGNED-NEXT: v_lshrrev_b32_e32 v86, 8, v86
-; ALIGNED-NEXT: flat_store_byte v[20:21], v31 offset:30
+; ALIGNED-NEXT: flat_store_byte v[20:21], v31 offset:34
; ALIGNED-NEXT: v_lshrrev_b32_e32 v31, 24, v80
; ALIGNED-NEXT: v_lshrrev_b32_e32 v80, 8, v80
; ALIGNED-NEXT: flat_store_byte v[20:21], v100 offset:28
@@ -7405,22 +7403,22 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:296
; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:300
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:296
-; ALIGNED-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:300
-; ALIGNED-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:288
-; ALIGNED-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT: buffer_load_dword v15, off, s[0:3], s32 offset:300
+; ALIGNED-NEXT: buffer_load_dword v14, off, s[0:3], s32 offset:296
+; ALIGNED-NEXT: buffer_load_dword v13, off, s[0:3], s32 offset:292
+; ALIGNED-NEXT: buffer_load_dword v12, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v15, off offset:42
-; ALIGNED-NEXT: global_store_byte v[0:1], v15, off offset:40
+; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v15, off offset:46
+; ALIGNED-NEXT: global_store_byte v[0:1], v15, off offset:44
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v14, off offset:46
-; ALIGNED-NEXT: global_store_byte v[0:1], v14, off offset:44
+; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v14, off offset:42
+; ALIGNED-NEXT: global_store_byte v[0:1], v14, off offset:40
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v13, off offset:34
-; ALIGNED-NEXT: global_store_byte v[0:1], v13, off offset:32
+; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v13, off offset:38
+; ALIGNED-NEXT: global_store_byte v[0:1], v13, off offset:36
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v12, off offset:38
-; ALIGNED-NEXT: global_store_byte v[0:1], v12, off offset:36
+; ALIGNED-NEXT: global_store_byte_d16_hi v[0:1], v12, off offset:34
+; ALIGNED-NEXT: global_store_byte v[0:1], v12, off offset:32
; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:244
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:248
@@ -7659,14 +7657,14 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; ALIGNED-NEXT: global_store_byte v[0:1], v97, off offset:53
; ALIGNED-NEXT: global_store_byte v[0:1], v38, off offset:51
; ALIGNED-NEXT: global_store_byte v[0:1], v96, off offset:49
-; ALIGNED-NEXT: global_store_byte v[0:1], v115, off offset:43
-; ALIGNED-NEXT: global_store_byte v[0:1], v15, off offset:41
-; ALIGNED-NEXT: global_store_byte v[0:1], v37, off offset:47
-; ALIGNED-NEXT: global_store_byte v[0:1], v14, off offset:45
-; ALIGNED-NEXT: global_store_byte v[0:1], v20, off offset:35
-; ALIGNED-NEXT: global_store_byte v[0:1], v13, off offset:33
-; ALIGNED-NEXT: global_store_byte v[0:1], v36, off offset:39
-; ALIGNED-NEXT: global_store_byte v[0:1], v12, off offset:37
+; ALIGNED-NEXT: global_store_byte v[0:1], v115, off offset:47
+; ALIGNED-NEXT: global_store_byte v[0:1], v15, off offset:45
+; ALIGNED-NEXT: global_store_byte v[0:1], v37, off offset:43
+; ALIGNED-NEXT: global_store_byte v[0:1], v14, off offset:41
+; ALIGNED-NEXT: global_store_byte v[0:1], v20, off offset:39
+; ALIGNED-NEXT: global_store_byte v[0:1], v13, off offset:37
+; ALIGNED-NEXT: global_store_byte v[0:1], v36, off offset:35
+; ALIGNED-NEXT: global_store_byte v[0:1], v12, off offset:33
; ALIGNED-NEXT: global_store_byte v[0:1], v100, off offset:31
; ALIGNED-NEXT: global_store_byte v[0:1], v11, off offset:29
; ALIGNED-NEXT: global_store_byte v[0:1], v51, off offset:27
@@ -8035,22 +8033,22 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:552
; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:556
; ALIGNED-NEXT: s_clause 0x3
-; ALIGNED-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:552
-; ALIGNED-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:556
-; ALIGNED-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:544
-; ALIGNED-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:548
+; ALIGNED-NEXT: buffer_load_dword v11, off, s[0:3], s32 offset:556
+; ALIGNED-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:552
+; ALIGNED-NEXT: buffer_load_dword v9, off, s[0:3], s32 offset:548
+; ALIGNED-NEXT: buffer_load_dword v8, off, s[0:3], s32 offset:544
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v11, off offset:42
-; ALIGNED-NEXT: global_store_byte v[12:13], v11, off offset:40
+; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v11, off offset:46
+; ALIGNED-NEXT: global_store_byte v[12:13], v11, off offset:44
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v10, off offset:46
-; ALIGNED-NEXT: global_store_byte v[12:13], v10, off offset:44
+; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v10, off offset:42
+; ALIGNED-NEXT: global_store_byte v[12:13], v10, off offset:40
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v9, off offset:34
-; ALIGNED-NEXT: global_store_byte v[12:13], v9, off offset:32
+; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v9, off offset:38
+; ALIGNED-NEXT: global_store_byte v[12:13], v9, off offset:36
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v8, off offset:38
-; ALIGNED-NEXT: global_store_byte v[12:13], v8, off offset:36
+; ALIGNED-NEXT: global_store_byte_d16_hi v[12:13], v8, off offset:34
+; ALIGNED-NEXT: global_store_byte v[12:13], v8, off offset:32
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:496
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:500
; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:504
@@ -8289,14 +8287,14 @@ define void @memmove_p1_p1_sz2048(ptr addrspace(1) align 1 %dst, ptr addrspace(1
; ALIGNED-NEXT: global_store_byte v[12:13], v18, off offset:53
; ALIGNED-NEXT: global_store_byte v[12:13], v84, off offset:51
; ALIGNED-NEXT: global_store_byte v[12:13], v16, off offset:49
-; ALIGNED-NEXT: global_store_byte v[12:13], v115, off offset:43
-; ALIGNED-NEXT: global_store_byte v[12:13], v11, off offset:41
-; ALIGNED-NEXT: global_store_byte v[12:13], v69, off offset:47
-; ALIGNED-NEXT: global_store_byte v[12:13], v10, off offset:45
-; ALIGNED-NEXT: global_store_byte v[12:13], v50, off offset:35
-; ALIGNED-NEXT: global_store_byte v[12:13], v9, off offset:33
-; ALIGNED-NEXT: global_store_byte v[12:13], v54, off offset:39
-; ALIGNED-NEXT: global_store_byte v[12:13], v8, off offset:37
+; ALIGNED-NEXT: global_store_byte v[12:13], v115, off offset:47
+; ALIGNED-NEXT: global_store_byte v[12:13], v11, off offset:45
+; ALIGNED-NEXT: global_store_byte v[12:13], v69, off offset:43
+; ALIGNED-NEXT: global_store_byte v[12:13], v10, off offset:41
+; ALIGNED-NEXT: global_store_byte v[12:13], v50, off offset:39
+; ALIGNED-NEXT: global_store_byte v[12:13], v9, off offset:37
+; ALIGNED-NEXT: global_store_byte v[12:13], v54, off offset:35
+; ALIGNED-NEXT: global_store_byte v[12:13], v8, off offset:33
; ALIGNED-NEXT: global_store_byte v[12:13], v100, off offset:31
; ALIGNED-NEXT: global_store_byte v[12:13], v7, off offset:29
; ALIGNED-NEXT: global_store_byte v[12:13], v85, off offset:27
@@ -8744,13 +8742,13 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v70, 24, v55
; ALIGNED-NEXT: v_lshrrev_b32_e32 v71, 8, v55
; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:200
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v52
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v53
; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:198
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v52
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v53
; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:204
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v113, 24, v53
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v113, 24, v52
; ALIGNED-NEXT: flat_store_byte v[98:99], v112 offset:202
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v112, 8, v53
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v112, 8, v52
; ALIGNED-NEXT: flat_store_byte v[98:99], v80 offset:196
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v80, 24, v50
@@ -8793,18 +8791,18 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v102, 24, v36
; ALIGNED-NEXT: flat_store_byte v[98:99], v103 offset:174
; ALIGNED-NEXT: v_lshrrev_b32_e32 v103, 8, v36
-; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:252
; ALIGNED-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:252
; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:244
; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v54 offset:164
-; ALIGNED-NEXT: flat_store_byte v[98:99], v54 offset:165
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v55 offset:168
; ALIGNED-NEXT: flat_store_byte v[98:99], v55 offset:169
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v52 offset:156
-; ALIGNED-NEXT: flat_store_byte v[98:99], v52 offset:157
+; ALIGNED-NEXT: flat_store_byte v[98:99], v54 offset:165
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v53 offset:160
; ALIGNED-NEXT: flat_store_byte v[98:99], v53 offset:161
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v52 offset:156
+; ALIGNED-NEXT: flat_store_byte v[98:99], v52 offset:157
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v52, 24, v34
; ALIGNED-NEXT: v_lshrrev_b32_e32 v53, 8, v34
@@ -8818,10 +8816,10 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v70, 24, v32
; ALIGNED-NEXT: flat_store_byte v[98:99], v71 offset:170
; ALIGNED-NEXT: v_lshrrev_b32_e32 v71, 8, v32
-; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:160
-; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:158
-; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:164
-; ALIGNED-NEXT: flat_store_byte v[98:99], v112 offset:162
+; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:164
+; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:162
+; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:160
+; ALIGNED-NEXT: flat_store_byte v[98:99], v112 offset:158
; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:200
; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:204
; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:196
@@ -9129,9 +9127,6 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: buffer_store_dword v101, off, s[0:3], s32 offset:292
; ALIGNED-NEXT: buffer_store_dword v100, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v102
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v102
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v113, 24, v103
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v112, 8, v103
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v102 offset:244
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v103 offset:248
; ALIGNED-NEXT: flat_store_byte v[98:99], v103 offset:249
@@ -9140,6 +9135,9 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: flat_store_byte v[98:99], v101 offset:241
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v100 offset:236
; ALIGNED-NEXT: flat_store_byte v[98:99], v100 offset:237
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v102
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v113, 24, v103
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v112, 8, v103
; ALIGNED-NEXT: v_lshrrev_b32_e32 v103, 24, v101
; ALIGNED-NEXT: v_lshrrev_b32_e32 v102, 8, v101
; ALIGNED-NEXT: v_lshrrev_b32_e32 v101, 24, v100
@@ -9165,6 +9163,9 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: buffer_store_dword v87, off, s[0:3], s32 offset:316
; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:308
; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:304
+; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:232
+; ALIGNED-NEXT: s_waitcnt vmcnt(13)
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v81
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v86 offset:228
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v87 offset:232
; ALIGNED-NEXT: flat_store_byte v[98:99], v87 offset:233
@@ -9173,13 +9174,10 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: flat_store_byte v[98:99], v85 offset:225
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v84 offset:220
; ALIGNED-NEXT: flat_store_byte v[98:99], v84 offset:221
-; ALIGNED-NEXT: s_waitcnt vmcnt(13)
; ALIGNED-NEXT: v_lshrrev_b32_e32 v84, 24, v82
; ALIGNED-NEXT: v_lshrrev_b32_e32 v85, 8, v82
; ALIGNED-NEXT: v_lshrrev_b32_e32 v86, 24, v83
; ALIGNED-NEXT: v_lshrrev_b32_e32 v87, 8, v83
-; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:232
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v81
; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:230
; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v81
; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:236
@@ -9211,8 +9209,10 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v69
; ALIGNED-NEXT: v_lshrrev_b32_e32 v82, 24, v68
; ALIGNED-NEXT: v_lshrrev_b32_e32 v83, 8, v68
-; ALIGNED-NEXT: flat_store_byte v[98:99], v84 offset:216
+; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:212
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v65
+; ALIGNED-NEXT: flat_store_byte v[98:99], v84 offset:216
; ALIGNED-NEXT: v_lshrrev_b32_e32 v84, 24, v66
; ALIGNED-NEXT: flat_store_byte v[98:99], v85 offset:214
; ALIGNED-NEXT: v_lshrrev_b32_e32 v85, 8, v66
@@ -9220,8 +9220,6 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v86, 24, v67
; ALIGNED-NEXT: flat_store_byte v[98:99], v87 offset:218
; ALIGNED-NEXT: v_lshrrev_b32_e32 v87, 8, v67
-; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:212
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v65
; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:210
; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v65
; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:208
@@ -9278,8 +9276,10 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v65, 8, v49
; ALIGNED-NEXT: v_lshrrev_b32_e32 v66, 24, v48
; ALIGNED-NEXT: v_lshrrev_b32_e32 v67, 8, v48
-; ALIGNED-NEXT: flat_store_byte v[98:99], v84 offset:184
+; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:180
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v37
+; ALIGNED-NEXT: flat_store_byte v[98:99], v84 offset:184
; ALIGNED-NEXT: v_lshrrev_b32_e32 v84, 24, v38
; ALIGNED-NEXT: flat_store_byte v[98:99], v85 offset:182
; ALIGNED-NEXT: v_lshrrev_b32_e32 v85, 8, v38
@@ -9287,8 +9287,6 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v86, 24, v39
; ALIGNED-NEXT: flat_store_byte v[98:99], v87 offset:186
; ALIGNED-NEXT: v_lshrrev_b32_e32 v87, 8, v39
-; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:180
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v37
; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:178
; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v37
; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:176
@@ -9359,6 +9357,10 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:348
; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:340
; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:336
+; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:132
+; ALIGNED-NEXT: s_waitcnt vmcnt(3)
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v18
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v26
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v38 offset:132
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v39 offset:136
; ALIGNED-NEXT: flat_store_byte v[98:99], v39 offset:137
@@ -9371,21 +9373,14 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: flat_store_byte v[98:99], v85 offset:134
; ALIGNED-NEXT: flat_store_byte v[98:99], v86 offset:140
; ALIGNED-NEXT: flat_store_byte v[98:99], v87 offset:138
-; ALIGNED-NEXT: flat_store_byte v[98:99], v115 offset:132
-; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v115, 24, v18
; ALIGNED-NEXT: flat_store_byte v[98:99], v114 offset:130
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v18
; ALIGNED-NEXT: flat_store_byte v[98:99], v113 offset:128
; ALIGNED-NEXT: flat_store_byte v[98:99], v112 offset:126
; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:424
; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:428
; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:420
; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:416
-; ALIGNED-NEXT: flat_store_byte v[98:99], v52 offset:120
-; ALIGNED-NEXT: s_waitcnt vmcnt(2)
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v52, 24, v14
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v81, 8, v26
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v114, 8, v18
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v34 offset:116
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v35 offset:120
; ALIGNED-NEXT: flat_store_byte v[98:99], v35 offset:121
@@ -9394,6 +9389,9 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: flat_store_byte v[98:99], v33 offset:113
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v32 offset:108
; ALIGNED-NEXT: flat_store_byte v[98:99], v32 offset:109
+; ALIGNED-NEXT: flat_store_byte v[98:99], v52 offset:120
+; ALIGNED-NEXT: s_waitcnt vmcnt(2)
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v52, 24, v14
; ALIGNED-NEXT: flat_store_byte v[98:99], v53 offset:118
; ALIGNED-NEXT: flat_store_byte v[98:99], v54 offset:124
; ALIGNED-NEXT: flat_store_byte v[98:99], v55 offset:122
@@ -9446,10 +9444,10 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: v_lshrrev_b32_e32 v34, 24, v16
; ALIGNED-NEXT: v_lshrrev_b32_e32 v35, 8, v16
; ALIGNED-NEXT: v_lshrrev_b32_e32 v55, 8, v15
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v68, 24, v12
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v69, 8, v12
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v70, 24, v13
-; ALIGNED-NEXT: v_lshrrev_b32_e32 v71, 8, v13
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v68, 24, v13
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v69, 8, v13
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v70, 24, v12
+; ALIGNED-NEXT: v_lshrrev_b32_e32 v71, 8, v12
; ALIGNED-NEXT: v_lshrrev_b32_e32 v31, 8, v11
; ALIGNED-NEXT: flat_store_byte v[98:99], v103 offset:104
; ALIGNED-NEXT: v_lshrrev_b32_e32 v103, 24, v9
@@ -9522,26 +9520,26 @@ define void @memmove_p0_p4_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(4
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v16 offset:44
; ALIGNED-NEXT: flat_store_byte v[98:99], v35 offset:46
; ALIGNED-NEXT: flat_store_byte v[98:99], v16 offset:45
-; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:508
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:504
+; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:508
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:500
; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:496
-; ALIGNED-NEXT: flat_store_byte v[98:99], v52 offset:40
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v14 offset:36
+; ALIGNED-NEXT: flat_store_byte v[98:99], v52 offset:40
; ALIGNED-NEXT: flat_store_byte v[98:99], v53 offset:38
-; ALIGNED-NEXT: flat_store_byte v[98:99], v14 offset:37
-; ALIGNED-NEXT: flat_store_byte v[98:99], v54 offset:44
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v15 offset:40
-; ALIGNED-NEXT: flat_store_byte v[98:99], v55 offset:42
+; ALIGNED-NEXT: flat_store_byte v[98:99], v54 offset:44
; ALIGNED-NEXT: flat_store_byte v[98:99], v15 offset:41
-; ALIGNED-NEXT: flat_store_byte v[98:99], v68 offset:32
-; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v12 offset:28
-; ALIGNED-NEXT: flat_store_byte v[98:99], v69 offset:30
-; ALIGNED-NEXT: flat_store_byte v[0:1], v12 offset:32
-; ALIGNED-NEXT: flat_store_byte v[98:99], v70 offset:36
+; ALIGNED-NEXT: flat_store_byte v[98:99], v55 offset:42
+; ALIGNED-NEXT: flat_store_byte v[98:99], v14 offset:37
+; ALIGNED-NEXT: flat_store_byte v[98:99], v68 offset:36
; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v13 offset:32
-; ALIGNED-NEXT: flat_store_byte v[98:99], v71 offset:34
+; ALIGNED-NEXT: flat_store_byte v[98:99], v69 offset:34
; ALIGNED-NEXT: flat_store_byte v[98:99], v13 offset:33
+; ALIGNED-NEXT: flat_store_byte v[98:99], v70 offset:32
+; ALIGNED-NEXT: flat_store_byte_d16_hi v[96:97], v12 offset:28
+; ALIGNED-NEXT: flat_store_byte v[98:99], v71 offset:30
+; ALIGNED-NEXT: flat_store_byte v[0:1], v12 offset:32
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:456
; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:460
; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:452
@@ -12762,11 +12760,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v125, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v126, off, s[0:3], s32 offset:4 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v127, off, s[0:3], s32 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_mov_b32_e32 v98, v1
-; ALIGNED-NEXT: v_mov_b32_e32 v97, v0
+; ALIGNED-NEXT: v_mov_b32_e32 v84, v1
+; ALIGNED-NEXT: v_mov_b32_e32 v83, v0
; ALIGNED-NEXT: s_mov_b32 s4, exec_lo
-; ALIGNED-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[97:98]
-; ALIGNED-NEXT: v_cndmask_b32_e32 v3, -1, v97, vcc_lo
+; ALIGNED-NEXT: v_cmp_ne_u64_e32 vcc_lo, 0, v[83:84]
+; ALIGNED-NEXT: v_cndmask_b32_e32 v3, -1, v83, vcc_lo
; ALIGNED-NEXT: v_cmpx_ge_u32_e32 v2, v3
; ALIGNED-NEXT: s_xor_b32 s6, exec_lo, s4
; ALIGNED-NEXT: s_cbranch_execz .LBB9_3
@@ -12782,43 +12780,43 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:24
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:25
; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:26
-; ALIGNED-NEXT: buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:19
+; ALIGNED-NEXT: buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:19
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:27
; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:28
; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:29
; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:30
; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:31
-; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:32
+; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:32
; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:33
-; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:34
-; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:35
-; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:36
-; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:37
-; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:38
+; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:34
+; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:35
+; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:36
+; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:37
+; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:38
; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:39
; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:40
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:41
-; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:42
-; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:43
+; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:41
+; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:42
; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:44
-; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:45
-; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:46
-; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:47
+; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:45
+; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:46
+; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:47
; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:48
; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:49
; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:50
+; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:43
; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:51
; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:52
-; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:53
-; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:54
-; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:55
-; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:56
+; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:53
+; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:54
+; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:55
+; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:56
; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:57
; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:58
; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:60
-; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:61
-; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:62
-; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:63
+; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:61
+; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:62
+; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:63
; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:64
; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:65
; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:66
@@ -12830,7 +12828,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:71
; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:72
; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:73
-; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:74
+; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:74
; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:76
; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:77
; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:78
@@ -12863,69 +12861,67 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(44)
-; ALIGNED-NEXT: v_lshl_or_b32 v9, v16, 8, v15
+; ALIGNED-NEXT: v_lshl_or_b32 v9, v16, 8, v14
; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 16, v5
; ALIGNED-NEXT: s_waitcnt vmcnt(42)
-; ALIGNED-NEXT: v_lshl_or_b32 v10, v20, 8, v18
+; ALIGNED-NEXT: v_lshl_or_b32 v10, v18, 8, v17
; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(40)
-; ALIGNED-NEXT: v_lshl_or_b32 v11, v17, 8, v14
+; ALIGNED-NEXT: v_lshl_or_b32 v11, v19, 8, v15
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v8, 16, v7
; ALIGNED-NEXT: s_waitcnt vmcnt(38)
-; ALIGNED-NEXT: v_lshl_or_b32 v12, v21, 8, v19
+; ALIGNED-NEXT: v_lshl_or_b32 v12, v21, 8, v20
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(36)
-; ALIGNED-NEXT: v_lshl_or_b32 v13, v24, 8, v23
+; ALIGNED-NEXT: s_waitcnt vmcnt(33)
+; ALIGNED-NEXT: v_lshl_or_b32 v13, v24, 8, v22
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v10, 16, v9
-; ALIGNED-NEXT: s_waitcnt vmcnt(34)
-; ALIGNED-NEXT: v_lshl_or_b32 v14, v28, 8, v26
-; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(32)
-; ALIGNED-NEXT: v_lshl_or_b32 v15, v25, 8, v22
+; ALIGNED-NEXT: s_waitcnt vmcnt(31)
+; ALIGNED-NEXT: v_lshl_or_b32 v14, v26, 8, v25
+; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v15, v27, 8, v23
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v12, 16, v11
-; ALIGNED-NEXT: s_waitcnt vmcnt(30)
-; ALIGNED-NEXT: v_lshl_or_b32 v16, v29, 8, v27
-; ALIGNED-NEXT: s_waitcnt vmcnt(28)
+; ALIGNED-NEXT: s_waitcnt vmcnt(27)
+; ALIGNED-NEXT: v_lshl_or_b32 v16, v28, 8, v29
; ALIGNED-NEXT: v_lshl_or_b32 v3, v31, 8, v30
; ALIGNED-NEXT: s_waitcnt vmcnt(26)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v34, 8, v33
; ALIGNED-NEXT: s_waitcnt vmcnt(24)
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v37, 8, v32
+; ALIGNED-NEXT: v_lshl_or_b32 v5, v35, 8, v32
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v14, 16, v13
; ALIGNED-NEXT: s_waitcnt vmcnt(22)
-; ALIGNED-NEXT: v_lshl_or_b32 v6, v36, 8, v35
+; ALIGNED-NEXT: v_lshl_or_b32 v6, v37, 8, v36
; ALIGNED-NEXT: s_waitcnt vmcnt(17)
-; ALIGNED-NEXT: v_lshl_or_b32 v7, v50, 8, v38
+; ALIGNED-NEXT: v_lshl_or_b32 v7, v39, 8, v38
; ALIGNED-NEXT: s_waitcnt vmcnt(15)
-; ALIGNED-NEXT: v_lshl_or_b32 v8, v49, 8, v39
-; ALIGNED-NEXT: v_lshl_or_b32 v9, v51, 8, v48
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v8, v50, 8, v48
+; ALIGNED-NEXT: v_lshl_or_b32 v9, v51, 8, v49
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v16, 16, v15
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v52, 8, v53
; ALIGNED-NEXT: v_lshl_or_b32 v4, v55, 8, v54
-; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 16, v5
; ALIGNED-NEXT: s_waitcnt vmcnt(10)
; ALIGNED-NEXT: v_lshl_or_b32 v5, v64, 8, v66
-; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v8, 16, v7
; ALIGNED-NEXT: s_clause 0x4
@@ -12938,39 +12934,39 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: v_lshl_or_b32 v0, v3, 16, v9
; ALIGNED-NEXT: s_waitcnt vmcnt(13)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v67, 8, v65
-; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v5, 16, v4
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v69, 8, v68
; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:81
-; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:964 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x1
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:79
-; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:75
+; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:75
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v71, 8, v70
; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:916 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
@@ -12986,8 +12982,8 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v80, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v81, off, s[0:3], s32 offset:980 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v82, off, s[0:3], s32 offset:984 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v125, off, s[0:3], s32 offset:1452 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:992 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v127, off, s[0:3], s32 offset:1452 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x4
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:91
; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:113
@@ -13008,10 +13004,10 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: v_lshl_or_b32 v4, v0, 8, v80
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:976 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: buffer_store_dword v83, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:988 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v82, 8, v81
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v83, 8, v84
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v85, 8, v86
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:996 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v5, 8, v1
@@ -13236,94 +13232,93 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1280 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x3e
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:141
-; ALIGNED-NEXT: buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:159
+; ALIGNED-NEXT: buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:159
; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:160
-; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:161
-; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:162
-; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:163
-; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:164
-; ALIGNED-NEXT: buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:165
-; ALIGNED-NEXT: buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:166
-; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:167
-; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:168
-; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:169
+; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:161
+; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:162
+; ALIGNED-NEXT: buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:163
+; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:164
+; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:165
+; ALIGNED-NEXT: buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:166
+; ALIGNED-NEXT: buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:167
+; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:168
+; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:169
; ALIGNED-NEXT: buffer_load_ubyte v114, v2, s[0:3], 0 offen offset:170
-; ALIGNED-NEXT: buffer_load_ubyte v116, v2, s[0:3], 0 offen offset:171
-; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:172
-; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:173
-; ALIGNED-NEXT: buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:174
-; ALIGNED-NEXT: buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:175
-; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:176
-; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:177
-; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:178
-; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:179
-; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:180
-; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:181
-; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:182
-; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:183
-; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:184
-; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:185
-; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:186
-; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:187
-; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:188
-; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:189
-; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:190
-; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:191
-; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:192
-; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:193
-; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:194
-; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:195
-; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:196
-; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:197
-; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:198
-; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:199
-; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:200
-; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:201
-; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:202
-; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:203
-; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:204
-; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:205
-; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:206
-; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:207
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:208
-; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:209
-; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:210
-; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:211
-; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:212
-; ALIGNED-NEXT: buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:229
-; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:230
-; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:231
-; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:232
-; ALIGNED-NEXT: buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:233
-; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:234
-; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:235
-; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:236
-; ALIGNED-NEXT: s_clause 0x18
-; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:237
-; ALIGNED-NEXT: buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:238
-; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:239
-; ALIGNED-NEXT: buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:240
-; ALIGNED-NEXT: buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:241
-; ALIGNED-NEXT: buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:242
-; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:243
-; ALIGNED-NEXT: buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:244
-; ALIGNED-NEXT: buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:245
-; ALIGNED-NEXT: buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:246
-; ALIGNED-NEXT: buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:247
-; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:248
-; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:249
-; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:250
-; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:251
-; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:252
-; ALIGNED-NEXT: buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:253
-; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:254
-; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:255
-; ALIGNED-NEXT: buffer_load_ubyte v96, v2, s[0:3], 0 offen
-; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:171
+; ALIGNED-NEXT: buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:172
+; ALIGNED-NEXT: buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:173
+; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:174
+; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:175
+; ALIGNED-NEXT: buffer_load_ubyte v100, v2, s[0:3], 0 offen offset:176
+; ALIGNED-NEXT: buffer_load_ubyte v80, v2, s[0:3], 0 offen offset:177
+; ALIGNED-NEXT: buffer_load_ubyte v71, v2, s[0:3], 0 offen offset:178
+; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:179
+; ALIGNED-NEXT: buffer_load_ubyte v81, v2, s[0:3], 0 offen offset:180
+; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:181
+; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:182
+; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:183
+; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:184
+; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:185
+; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:186
+; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:187
+; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:188
+; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:189
+; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:190
+; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:191
+; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:192
+; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:193
+; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:194
+; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:195
+; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:196
+; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:197
+; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:198
+; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:199
+; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:200
+; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:201
+; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:202
+; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:203
+; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:204
+; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:205
+; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:206
+; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:207
+; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:208
+; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:209
+; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:210
+; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:211
+; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:212
+; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:229
+; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:230
+; ALIGNED-NEXT: buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:231
+; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:232
+; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:233
+; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:234
+; ALIGNED-NEXT: buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:235
+; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:236
+; ALIGNED-NEXT: s_clause 0x17
+; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:237
+; ALIGNED-NEXT: buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:238
+; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:239
+; ALIGNED-NEXT: buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:240
+; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:241
+; ALIGNED-NEXT: buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:242
+; ALIGNED-NEXT: buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:243
+; ALIGNED-NEXT: buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:244
+; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:245
+; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:246
+; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:247
+; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:248
+; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:249
+; ALIGNED-NEXT: buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:250
+; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:251
+; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:252
+; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:253
+; ALIGNED-NEXT: buffer_load_ubyte v99, v2, s[0:3], 0 offen offset:254
+; ALIGNED-NEXT: buffer_load_ubyte v96, v2, s[0:3], 0 offen offset:255
+; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen
; ALIGNED-NEXT: buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:4
-; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:5
+; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:5
; ALIGNED-NEXT: buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:6
-; ALIGNED-NEXT: buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:7
+; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:7
; ALIGNED-NEXT: s_waitcnt vmcnt(62)
; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1308 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1316 ; 4-byte Folded Spill
@@ -13354,285 +13349,289 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:1336 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:1332 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1320 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_clause 0x7
-; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:213
-; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:214
-; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:215
-; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:216
-; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:217
-; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:218
-; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:219
-; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:220
+; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1356 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_load_ubyte v4, v2, s[0:3], 0 offen offset:151
-; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1348 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(29)
-; ALIGNED-NEXT: v_lshl_or_b32 v117, v46, 8, v56
-; ALIGNED-NEXT: s_waitcnt vmcnt(21)
-; ALIGNED-NEXT: v_lshl_or_b32 v45, v102, 8, v112
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1340 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(20)
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v40, 8, v42
+; ALIGNED-NEXT: s_waitcnt vmcnt(12)
+; ALIGNED-NEXT: v_lshl_or_b32 v47, v96, 8, v99
+; ALIGNED-NEXT: v_lshl_or_b32 v98, v117, 8, v119
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1344 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:149
-; ALIGNED-NEXT: s_waitcnt vmcnt(16)
-; ALIGNED-NEXT: buffer_store_dword v88, off, s[0:3], s32 offset:1380 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(8)
+; ALIGNED-NEXT: buffer_store_dword v95, off, s[0:3], s32 offset:1372 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v105, off, s[0:3], s32 offset:1388 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v73, off, s[0:3], s32 offset:1368 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v77, off, s[0:3], s32 offset:1372 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v61, off, s[0:3], s32 offset:1344 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(15)
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1360 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(14)
+; ALIGNED-NEXT: buffer_store_dword v72, off, s[0:3], s32 offset:1360 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v77, off, s[0:3], s32 offset:1364 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(7)
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1380 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(6)
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1404 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(13)
+; ALIGNED-NEXT: s_waitcnt vmcnt(5)
; ALIGNED-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:1420 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(12)
+; ALIGNED-NEXT: s_waitcnt vmcnt(4)
; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1440 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(11)
-; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1428 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(10)
-; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1432 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(3)
+; ALIGNED-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:1432 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(2)
+; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1436 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1376 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:1384 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v4, v4, 8, v1
; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:158
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v10
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1356 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1368 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1396 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:157
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v127, 8, v1
-; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v125, 8, v1
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v5
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v6
; ALIGNED-NEXT: v_lshl_or_b32 v4, v9, 8, v8
+; ALIGNED-NEXT: s_clause 0x7
+; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:213
+; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:214
+; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:215
+; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:216
+; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:217
+; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:218
+; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:219
+; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:220
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v92, 8, v107
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v75, 8, v79
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v79, 8, v107
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v74, 8, v120
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v59, 8, v95
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v47, 8, v57
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v61, 8, v93
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v56, 8, v59
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v101, 8, v113
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v87, 8, v103
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v88, 8, v103
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v86, 8, v102
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1460 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v42, 8, v111
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v116, 8, v114
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v43, 8, v45
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v41, 8, v114
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v84, 8, v86
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v82, 8, v83
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v80, 8, v100
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v70, 8, v71
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v71, 8, v85
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v69, 8, v70
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v67, 8, v81
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v66, 8, v68
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1472 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v55, 8, v64
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v53, 8, v54
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v51, 8, v53
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v49, 8, v50
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1476 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v67, 8, v68
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v66, 8, v65
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v64, 8, v65
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v54, 8, v55
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v49, 8, v52
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v48, 8, v39
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v38, 8, v48
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v36, 8, v37
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x7
-; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:221
-; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:222
-; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:223
-; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:224
-; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:225
-; ALIGNED-NEXT: buffer_load_ubyte v126, v2, s[0:3], 0 offen offset:226
+; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:221
+; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:222
+; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:223
+; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:224
+; ALIGNED-NEXT: buffer_load_ubyte v126, v2, s[0:3], 0 offen offset:225
+; ALIGNED-NEXT: buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:226
; ALIGNED-NEXT: buffer_load_ubyte v124, v2, s[0:3], 0 offen offset:227
-; ALIGNED-NEXT: buffer_load_ubyte v1, v2, s[0:3], 0 offen offset:228
+; ALIGNED-NEXT: buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:228
; ALIGNED-NEXT: v_lshl_or_b32 v63, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v37, 8, v50
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v35, 8, v36
-; ALIGNED-NEXT: v_lshl_or_b32 v119, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v27, 8, v29
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v25, 8, v26
-; ALIGNED-NEXT: v_lshl_or_b32 v81, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v32, 8, v34
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v30, 8, v31
-; ALIGNED-NEXT: v_lshl_or_b32 v80, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v14, 8, v19
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v11, 8, v15
-; ALIGNED-NEXT: v_lshl_or_b32 v51, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v16, 8, v18
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v13, 8, v12
-; ALIGNED-NEXT: v_lshl_or_b32 v38, v4, 16, v3
-; ALIGNED-NEXT: s_waitcnt vmcnt(7)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v9, 8, v17
-; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v7, 8, v8
-; ALIGNED-NEXT: v_lshl_or_b32 v33, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v22, 8, v24
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v21, 8, v23
-; ALIGNED-NEXT: v_lshl_or_b32 v28, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v33, 8, v39
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v32, 8, v34
+; ALIGNED-NEXT: v_lshl_or_b32 v116, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v23, 8, v26
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v22, 8, v24
+; ALIGNED-NEXT: v_lshl_or_b32 v82, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v29, 8, v30
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v28, 8, v27
+; ALIGNED-NEXT: v_lshl_or_b32 v69, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v19, 8, v21
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v18, 8, v17
+; ALIGNED-NEXT: v_lshl_or_b32 v52, v4, 16, v3
+; ALIGNED-NEXT: s_waitcnt vmcnt(15)
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v15, 8, v20
+; ALIGNED-NEXT: s_waitcnt vmcnt(13)
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v13, 8, v14
+; ALIGNED-NEXT: v_lshl_or_b32 v35, v4, 16, v3
+; ALIGNED-NEXT: s_waitcnt vmcnt(7)
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v7, 8, v8
+; ALIGNED-NEXT: s_waitcnt vmcnt(5)
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v5, 8, v6
+; ALIGNED-NEXT: v_lshl_or_b32 v31, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v11, 8, v12
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v10, 8, v9
+; ALIGNED-NEXT: v_lshl_or_b32 v25, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v6
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v126, 8, v0
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v124, 8, v126
-; ALIGNED-NEXT: v_lshl_or_b32 v20, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v124, 8, v123
+; ALIGNED-NEXT: v_lshl_or_b32 v16, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v122, 8, v1
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v110, 8, v120
-; ALIGNED-NEXT: v_lshl_or_b32 v10, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v89, 8, v94
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v91, 8, v90
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v106, 8, v108
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v93, 8, v104
-; ALIGNED-NEXT: v_lshl_or_b32 v60, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v76, 8, v78
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v72, 8, v74
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v110, 8, v121
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v106, 8, v108
+; ALIGNED-NEXT: v_lshl_or_b32 v1, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v92, 8, v91
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v75, 8, v78
+; ALIGNED-NEXT: v_lshl_or_b32 v58, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v94, 8, v104
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v90, 8, v89
+; ALIGNED-NEXT: v_lshl_or_b32 v46, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v73, 8, v76
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v60, 8, v62
; ALIGNED-NEXT: v_lshl_or_b32 v3, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v58, 8, v62
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v117, 16, v4
-; ALIGNED-NEXT: v_lshl_or_b32 v117, v115, 8, v118
-; ALIGNED-NEXT: v_lshl_or_b32 v99, v45, 16, v117
-; ALIGNED-NEXT: v_lshl_or_b32 v117, v44, 8, v43
-; ALIGNED-NEXT: v_lshl_or_b32 v45, v41, 8, v40
-; ALIGNED-NEXT: v_lshl_or_b32 v100, v45, 16, v117
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v44, 8, v57
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v118, 16, v4
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v101, 8, v113
+; ALIGNED-NEXT: v_lshl_or_b32 v97, v47, 16, v118
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v112, 8, v115
+; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:3
+; ALIGNED-NEXT: v_lshl_or_b32 v87, v118, 16, v98
; ALIGNED-NEXT: s_clause 0x1
-; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:1
-; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:2
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1352 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:1
+; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:2
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:1348 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(2)
+; ALIGNED-NEXT: buffer_store_dword v47, off, s[0:3], s32 offset:1340 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: buffer_store_dword v117, off, s[0:3], s32 offset:1364 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v45, off, s[0:3], s32 offset:1384 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v117, v117, 8, v96
-; ALIGNED-NEXT: v_lshl_or_b32 v45, v61, 8, v45
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v45, 16, v117
-; ALIGNED-NEXT: v_lshl_or_b32 v117, v73, 8, v105
-; ALIGNED-NEXT: v_lshl_or_b32 v45, v88, 8, v77
+; ALIGNED-NEXT: buffer_store_dword v98, off, s[0:3], s32 offset:1352 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: buffer_store_dword v118, off, s[0:3], s32 offset:1376 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v98, v98, 8, v85
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v47, 8, v118
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v118, 16, v98
+; ALIGNED-NEXT: v_lshl_or_b32 v98, v72, 8, v105
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v95, 8, v77
; ALIGNED-NEXT: s_clause 0x6
-; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:13
-; ALIGNED-NEXT: buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:14
-; ALIGNED-NEXT: buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:15
-; ALIGNED-NEXT: buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:11
+; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:13
+; ALIGNED-NEXT: buffer_load_ubyte v122, v2, s[0:3], 0 offen offset:14
+; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:15
+; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:11
; ALIGNED-NEXT: buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:8
; ALIGNED-NEXT: buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:9
-; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:10
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1392 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v45, 16, v117
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_load_ubyte v96, v2, s[0:3], 0 offen offset:12
+; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:10
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:1392 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v118, 16, v98
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_load_ubyte v85, v2, s[0:3], 0 offen offset:12
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: v_lshl_or_b32 v45, v121, 8, v123
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v117, v109, 8, v96
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v45, 16, v117
-; ALIGNED-NEXT: v_lshl_or_b32 v117, v77, 8, v105
-; ALIGNED-NEXT: v_lshl_or_b32 v45, v88, 8, v73
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1436 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v45, 16, v117
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v109, 8, v122
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: v_lshl_or_b32 v98, v111, 8, v85
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v118, 16, v98
+; ALIGNED-NEXT: v_lshl_or_b32 v98, v77, 8, v105
+; ALIGNED-NEXT: v_lshl_or_b32 v118, v95, 8, v72
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:1428 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v118, 16, v98
; ALIGNED-NEXT: s_clause 0x2
-; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:18
-; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:16
-; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:17
+; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:18
+; ALIGNED-NEXT: buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:16
+; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:17
; ALIGNED-NEXT: v_add_nc_u32_e32 v2, 0x100, v2
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v100, off, s[0:3], s32 offset:232
-; ALIGNED-NEXT: buffer_store_dword v99, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v87, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT: buffer_store_dword v97, off, s[0:3], s32 offset:236
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:224
-; ALIGNED-NEXT: v_add_co_u32 v3, vcc_lo, v97, 3
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v98, vcc_lo
-; ALIGNED-NEXT: flat_store_byte v[3:4], v40 offset:247
-; ALIGNED-NEXT: flat_store_byte v[3:4], v41 offset:248
-; ALIGNED-NEXT: flat_store_byte v[3:4], v44 offset:246
-; ALIGNED-NEXT: flat_store_byte v[3:4], v102 offset:252
-; ALIGNED-NEXT: flat_store_byte v[3:4], v115 offset:250
-; ALIGNED-NEXT: flat_store_byte v[3:4], v112 offset:251
-; ALIGNED-NEXT: flat_store_byte v[3:4], v118 offset:249
-; ALIGNED-NEXT: flat_store_byte v[3:4], v43 offset:245
-; ALIGNED-NEXT: flat_store_byte v[3:4], v74 offset:239
-; ALIGNED-NEXT: flat_store_byte v[3:4], v72 offset:240
-; ALIGNED-NEXT: flat_store_byte v[3:4], v76 offset:238
-; ALIGNED-NEXT: flat_store_byte v[3:4], v46 offset:244
-; ALIGNED-NEXT: flat_store_byte v[3:4], v58 offset:242
-; ALIGNED-NEXT: flat_store_byte v[3:4], v56 offset:243
-; ALIGNED-NEXT: flat_store_byte v[3:4], v62 offset:241
-; ALIGNED-NEXT: flat_store_byte v[3:4], v78 offset:237
-; ALIGNED-NEXT: buffer_store_dword v60, off, s[0:3], s32 offset:248
-; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:252
-; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:244
-; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:240
-; ALIGNED-NEXT: flat_store_byte v[3:4], v104 offset:231
-; ALIGNED-NEXT: flat_store_byte v[3:4], v93 offset:232
-; ALIGNED-NEXT: flat_store_byte v[3:4], v106 offset:230
-; ALIGNED-NEXT: flat_store_byte v[3:4], v91 offset:236
-; ALIGNED-NEXT: flat_store_byte v[3:4], v89 offset:234
-; ALIGNED-NEXT: flat_store_byte v[3:4], v90 offset:235
-; ALIGNED-NEXT: flat_store_byte v[3:4], v94 offset:233
-; ALIGNED-NEXT: flat_store_byte v[3:4], v108 offset:229
-; ALIGNED-NEXT: flat_store_byte v[3:4], v126 offset:223
+; ALIGNED-NEXT: v_add_co_u32 v3, vcc_lo, v83, 3
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v84, vcc_lo
+; ALIGNED-NEXT: flat_store_byte v[3:4], v115 offset:247
+; ALIGNED-NEXT: flat_store_byte v[3:4], v112 offset:248
+; ALIGNED-NEXT: flat_store_byte v[3:4], v117 offset:246
+; ALIGNED-NEXT: flat_store_byte v[3:4], v96 offset:252
+; ALIGNED-NEXT: flat_store_byte v[3:4], v101 offset:250
+; ALIGNED-NEXT: flat_store_byte v[3:4], v99 offset:251
+; ALIGNED-NEXT: flat_store_byte v[3:4], v113 offset:249
+; ALIGNED-NEXT: flat_store_byte v[3:4], v119 offset:245
+; ALIGNED-NEXT: flat_store_byte v[3:4], v62 offset:239
+; ALIGNED-NEXT: flat_store_byte v[3:4], v60 offset:240
+; ALIGNED-NEXT: flat_store_byte v[3:4], v73 offset:238
+; ALIGNED-NEXT: flat_store_byte v[3:4], v40 offset:244
+; ALIGNED-NEXT: flat_store_byte v[3:4], v44 offset:242
+; ALIGNED-NEXT: flat_store_byte v[3:4], v42 offset:243
+; ALIGNED-NEXT: flat_store_byte v[3:4], v57 offset:241
+; ALIGNED-NEXT: flat_store_byte v[3:4], v76 offset:237
+; ALIGNED-NEXT: buffer_store_dword v46, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:240
+; ALIGNED-NEXT: flat_store_byte v[3:4], v89 offset:231
+; ALIGNED-NEXT: flat_store_byte v[3:4], v90 offset:232
+; ALIGNED-NEXT: flat_store_byte v[3:4], v94 offset:230
+; ALIGNED-NEXT: flat_store_byte v[3:4], v75 offset:236
+; ALIGNED-NEXT: flat_store_byte v[3:4], v92 offset:234
+; ALIGNED-NEXT: flat_store_byte v[3:4], v78 offset:235
+; ALIGNED-NEXT: flat_store_byte v[3:4], v91 offset:233
+; ALIGNED-NEXT: flat_store_byte v[3:4], v104 offset:229
+; ALIGNED-NEXT: flat_store_byte v[3:4], v123 offset:223
; ALIGNED-NEXT: flat_store_byte v[3:4], v124 offset:224
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:222
-; ALIGNED-NEXT: flat_store_byte v[3:4], v110 offset:228
-; ALIGNED-NEXT: flat_store_byte v[3:4], v122 offset:226
-; ALIGNED-NEXT: flat_store_byte v[3:4], v120 offset:227
-; ALIGNED-NEXT: flat_store_byte v[3:4], v1 offset:225
-; ALIGNED-NEXT: flat_store_byte v[3:4], v6 offset:221
-; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:192
-; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:204
-; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:200
-; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:196
-; ALIGNED-NEXT: flat_store_byte v[3:4], v22 offset:206
-; ALIGNED-NEXT: flat_store_byte v[3:4], v21 offset:208
-; ALIGNED-NEXT: flat_store_byte v[3:4], v23 offset:207
-; ALIGNED-NEXT: flat_store_byte v[3:4], v14 offset:210
-; ALIGNED-NEXT: flat_store_byte v[3:4], v11 offset:212
-; ALIGNED-NEXT: flat_store_byte v[3:4], v15 offset:211
-; ALIGNED-NEXT: flat_store_byte v[3:4], v19 offset:209
-; ALIGNED-NEXT: flat_store_byte v[3:4], v12 offset:215
-; ALIGNED-NEXT: flat_store_byte v[3:4], v13 offset:216
-; ALIGNED-NEXT: flat_store_byte v[3:4], v16 offset:214
-; ALIGNED-NEXT: flat_store_byte v[3:4], v7 offset:220
-; ALIGNED-NEXT: flat_store_byte v[3:4], v9 offset:218
-; ALIGNED-NEXT: flat_store_byte v[3:4], v8 offset:219
-; ALIGNED-NEXT: flat_store_byte v[3:4], v17 offset:217
-; ALIGNED-NEXT: flat_store_byte v[3:4], v18 offset:213
-; ALIGNED-NEXT: flat_store_byte v[3:4], v24 offset:205
-; ALIGNED-NEXT: buffer_store_dword v80, off, s[0:3], s32 offset:216
-; ALIGNED-NEXT: buffer_store_dword v81, off, s[0:3], s32 offset:220
-; ALIGNED-NEXT: buffer_store_dword v119, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT: flat_store_byte v[3:4], v126 offset:222
+; ALIGNED-NEXT: flat_store_byte v[3:4], v106 offset:228
+; ALIGNED-NEXT: flat_store_byte v[3:4], v110 offset:226
+; ALIGNED-NEXT: flat_store_byte v[3:4], v108 offset:227
+; ALIGNED-NEXT: flat_store_byte v[3:4], v121 offset:225
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:221
+; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT: flat_store_byte v[3:4], v9 offset:215
+; ALIGNED-NEXT: flat_store_byte v[3:4], v10 offset:216
+; ALIGNED-NEXT: flat_store_byte v[3:4], v11 offset:214
+; ALIGNED-NEXT: flat_store_byte v[3:4], v5 offset:220
+; ALIGNED-NEXT: flat_store_byte v[3:4], v7 offset:218
+; ALIGNED-NEXT: flat_store_byte v[3:4], v6 offset:219
+; ALIGNED-NEXT: flat_store_byte v[3:4], v8 offset:217
+; ALIGNED-NEXT: flat_store_byte v[3:4], v12 offset:213
+; ALIGNED-NEXT: flat_store_byte v[3:4], v17 offset:207
+; ALIGNED-NEXT: flat_store_byte v[3:4], v18 offset:208
+; ALIGNED-NEXT: flat_store_byte v[3:4], v19 offset:206
+; ALIGNED-NEXT: flat_store_byte v[3:4], v13 offset:212
+; ALIGNED-NEXT: flat_store_byte v[3:4], v15 offset:210
+; ALIGNED-NEXT: flat_store_byte v[3:4], v14 offset:211
+; ALIGNED-NEXT: flat_store_byte v[3:4], v20 offset:209
+; ALIGNED-NEXT: flat_store_byte v[3:4], v21 offset:205
+; ALIGNED-NEXT: buffer_store_dword v69, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT: buffer_store_dword v82, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT: buffer_store_dword v116, off, s[0:3], s32 offset:212
; ALIGNED-NEXT: buffer_store_dword v63, off, s[0:3], s32 offset:208
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v31 offset:199
-; ALIGNED-NEXT: flat_store_byte v[3:4], v30 offset:200
-; ALIGNED-NEXT: flat_store_byte v[3:4], v32 offset:198
-; ALIGNED-NEXT: flat_store_byte v[3:4], v25 offset:204
-; ALIGNED-NEXT: flat_store_byte v[3:4], v27 offset:202
-; ALIGNED-NEXT: flat_store_byte v[3:4], v26 offset:203
-; ALIGNED-NEXT: flat_store_byte v[3:4], v29 offset:201
-; ALIGNED-NEXT: flat_store_byte v[3:4], v34 offset:197
-; ALIGNED-NEXT: flat_store_byte v[3:4], v39 offset:191
-; ALIGNED-NEXT: flat_store_byte v[3:4], v48 offset:192
-; ALIGNED-NEXT: flat_store_byte v[3:4], v49 offset:190
-; ALIGNED-NEXT: flat_store_byte v[3:4], v35 offset:196
-; ALIGNED-NEXT: flat_store_byte v[3:4], v37 offset:194
-; ALIGNED-NEXT: flat_store_byte v[3:4], v36 offset:195
-; ALIGNED-NEXT: flat_store_byte v[3:4], v50 offset:193
-; ALIGNED-NEXT: flat_store_byte v[3:4], v52 offset:189
+; ALIGNED-NEXT: flat_store_byte v[3:4], v27 offset:199
+; ALIGNED-NEXT: flat_store_byte v[3:4], v28 offset:200
+; ALIGNED-NEXT: flat_store_byte v[3:4], v29 offset:198
+; ALIGNED-NEXT: flat_store_byte v[3:4], v22 offset:204
+; ALIGNED-NEXT: flat_store_byte v[3:4], v23 offset:202
+; ALIGNED-NEXT: flat_store_byte v[3:4], v24 offset:203
+; ALIGNED-NEXT: flat_store_byte v[3:4], v26 offset:201
+; ALIGNED-NEXT: flat_store_byte v[3:4], v30 offset:197
+; ALIGNED-NEXT: flat_store_byte v[3:4], v37 offset:191
+; ALIGNED-NEXT: flat_store_byte v[3:4], v36 offset:192
+; ALIGNED-NEXT: flat_store_byte v[3:4], v38 offset:190
+; ALIGNED-NEXT: flat_store_byte v[3:4], v32 offset:196
+; ALIGNED-NEXT: flat_store_byte v[3:4], v33 offset:194
+; ALIGNED-NEXT: flat_store_byte v[3:4], v34 offset:195
+; ALIGNED-NEXT: flat_store_byte v[3:4], v39 offset:193
+; ALIGNED-NEXT: flat_store_byte v[3:4], v48 offset:189
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v125, 8, v117
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v127, 8, v118
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v125, v45, 8, v61
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v96, 16, v125
+; ALIGNED-NEXT: v_lshl_or_b32 v127, v47, 8, v98
+; ALIGNED-NEXT: v_lshl_or_b32 v85, v85, 16, v127
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:296
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
@@ -13645,22 +13644,22 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:288
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v65 offset:183
-; ALIGNED-NEXT: flat_store_byte v[3:4], v66 offset:184
-; ALIGNED-NEXT: flat_store_byte v[3:4], v67 offset:182
-; ALIGNED-NEXT: flat_store_byte v[3:4], v53 offset:188
-; ALIGNED-NEXT: flat_store_byte v[3:4], v55 offset:186
-; ALIGNED-NEXT: flat_store_byte v[3:4], v54 offset:187
-; ALIGNED-NEXT: flat_store_byte v[3:4], v64 offset:185
-; ALIGNED-NEXT: flat_store_byte v[3:4], v68 offset:181
-; ALIGNED-NEXT: flat_store_byte v[3:4], v83 offset:175
-; ALIGNED-NEXT: flat_store_byte v[3:4], v82 offset:176
-; ALIGNED-NEXT: flat_store_byte v[3:4], v84 offset:174
-; ALIGNED-NEXT: flat_store_byte v[3:4], v69 offset:180
-; ALIGNED-NEXT: flat_store_byte v[3:4], v71 offset:178
-; ALIGNED-NEXT: flat_store_byte v[3:4], v70 offset:179
-; ALIGNED-NEXT: flat_store_byte v[3:4], v85 offset:177
-; ALIGNED-NEXT: flat_store_byte v[3:4], v86 offset:173
+; ALIGNED-NEXT: flat_store_byte v[3:4], v55 offset:183
+; ALIGNED-NEXT: flat_store_byte v[3:4], v54 offset:184
+; ALIGNED-NEXT: flat_store_byte v[3:4], v64 offset:182
+; ALIGNED-NEXT: flat_store_byte v[3:4], v49 offset:188
+; ALIGNED-NEXT: flat_store_byte v[3:4], v51 offset:186
+; ALIGNED-NEXT: flat_store_byte v[3:4], v50 offset:187
+; ALIGNED-NEXT: flat_store_byte v[3:4], v53 offset:185
+; ALIGNED-NEXT: flat_store_byte v[3:4], v65 offset:181
+; ALIGNED-NEXT: flat_store_byte v[3:4], v71 offset:175
+; ALIGNED-NEXT: flat_store_byte v[3:4], v70 offset:176
+; ALIGNED-NEXT: flat_store_byte v[3:4], v80 offset:174
+; ALIGNED-NEXT: flat_store_byte v[3:4], v66 offset:180
+; ALIGNED-NEXT: flat_store_byte v[3:4], v67 offset:178
+; ALIGNED-NEXT: flat_store_byte v[3:4], v68 offset:179
+; ALIGNED-NEXT: flat_store_byte v[3:4], v81 offset:177
+; ALIGNED-NEXT: flat_store_byte v[3:4], v100 offset:173
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:312
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Reload
@@ -13672,22 +13671,22 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:304
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Reload
; ALIGNED-NEXT: flat_store_byte v[3:4], v114 offset:167
-; ALIGNED-NEXT: flat_store_byte v[3:4], v116 offset:168
-; ALIGNED-NEXT: flat_store_byte v[3:4], v42 offset:166
-; ALIGNED-NEXT: flat_store_byte v[3:4], v87 offset:172
-; ALIGNED-NEXT: flat_store_byte v[3:4], v101 offset:170
-; ALIGNED-NEXT: flat_store_byte v[3:4], v103 offset:171
-; ALIGNED-NEXT: flat_store_byte v[3:4], v113 offset:169
-; ALIGNED-NEXT: flat_store_byte v[3:4], v111 offset:165
-; ALIGNED-NEXT: flat_store_byte v[3:4], v79 offset:159
-; ALIGNED-NEXT: flat_store_byte v[3:4], v75 offset:160
-; ALIGNED-NEXT: flat_store_byte v[3:4], v92 offset:158
-; ALIGNED-NEXT: flat_store_byte v[3:4], v47 offset:164
-; ALIGNED-NEXT: flat_store_byte v[3:4], v59 offset:162
-; ALIGNED-NEXT: flat_store_byte v[3:4], v57 offset:163
-; ALIGNED-NEXT: flat_store_byte v[3:4], v95 offset:161
+; ALIGNED-NEXT: flat_store_byte v[3:4], v41 offset:168
+; ALIGNED-NEXT: flat_store_byte v[3:4], v43 offset:166
+; ALIGNED-NEXT: flat_store_byte v[3:4], v86 offset:172
+; ALIGNED-NEXT: flat_store_byte v[3:4], v88 offset:170
+; ALIGNED-NEXT: flat_store_byte v[3:4], v102 offset:171
+; ALIGNED-NEXT: flat_store_byte v[3:4], v103 offset:169
+; ALIGNED-NEXT: flat_store_byte v[3:4], v45 offset:165
+; ALIGNED-NEXT: flat_store_byte v[3:4], v120 offset:159
+; ALIGNED-NEXT: flat_store_byte v[3:4], v74 offset:160
+; ALIGNED-NEXT: flat_store_byte v[3:4], v79 offset:158
+; ALIGNED-NEXT: flat_store_byte v[3:4], v56 offset:164
+; ALIGNED-NEXT: flat_store_byte v[3:4], v61 offset:162
+; ALIGNED-NEXT: flat_store_byte v[3:4], v59 offset:163
+; ALIGNED-NEXT: flat_store_byte v[3:4], v93 offset:161
; ALIGNED-NEXT: flat_store_byte v[3:4], v107 offset:157
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:264
@@ -13697,23 +13696,23 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1396 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:260
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1340 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1344 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:256
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:151
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1432 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1436 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:152
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1428 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1432 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:150
-; ALIGNED-NEXT: flat_store_byte v[3:4], v127 offset:156
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v125 offset:156
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:154
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:155
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1404 ; 4-byte Folded Reload
@@ -13731,16 +13730,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1328 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:142
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1376 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1384 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:148
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1356 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1368 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:146
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1360 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1380 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:147
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1348 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1356 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:145
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1324 ; 4-byte Folded Reload
@@ -13805,7 +13804,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:129
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1244 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[97:98], v0 offset:128
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0 offset:128
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1240 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:360
@@ -14045,7 +14044,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:65
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:916 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[97:98], v0 offset:64
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0 offset:64
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:928 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:424
@@ -14055,13 +14054,10 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:420
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:416
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:58
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:55
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
@@ -14075,29 +14071,32 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:60
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:58
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:59
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:876 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:57
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:53
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:50
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:47
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:48
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:46
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:52
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:844 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:50
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:51
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
@@ -14108,10 +14107,10 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:45
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:440
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:444
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:436
@@ -14120,52 +14119,52 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:432
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:40
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:39
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:38
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:37
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:40
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:44
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:38
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:43
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:44
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:42
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:43
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:41
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:32
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:37
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:31
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:30
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[97:98], v0 offset:32
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:30
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:36
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:34
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:35
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:34
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:33
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:33
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:392
@@ -14175,7 +14174,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:388
-; ALIGNED-NEXT: buffer_store_dword v96, off, s[0:3], s32 offset:384
+; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:384
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:23
@@ -14200,11 +14199,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:21
-; ALIGNED-NEXT: flat_store_byte v[3:4], v117 offset:15
+; ALIGNED-NEXT: flat_store_byte v[3:4], v118 offset:15
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1452 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:16
-; ALIGNED-NEXT: flat_store_byte v[3:4], v45 offset:14
+; ALIGNED-NEXT: flat_store_byte v[3:4], v47 offset:14
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:716 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:20
@@ -14217,11 +14216,11 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:17
-; ALIGNED-NEXT: flat_store_byte v[97:98], v61 offset:16
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[83:84], v98 offset:16
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:408
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1436 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1428 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:412
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1400 ; 4-byte Folded Reload
@@ -14230,40 +14229,40 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1392 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:400
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v73 offset:7
-; ALIGNED-NEXT: flat_store_byte v[3:4], v88 offset:8
-; ALIGNED-NEXT: flat_store_byte v[3:4], v109 offset:10
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v72 offset:7
+; ALIGNED-NEXT: flat_store_byte v[3:4], v95 offset:8
; ALIGNED-NEXT: flat_store_byte v[3:4], v77 offset:6
-; ALIGNED-NEXT: flat_store_byte v[3:4], v121 offset:12
-; ALIGNED-NEXT: flat_store_byte v[3:4], v123 offset:11
+; ALIGNED-NEXT: flat_store_byte v[3:4], v109 offset:12
+; ALIGNED-NEXT: flat_store_byte v[3:4], v111 offset:10
+; ALIGNED-NEXT: flat_store_byte v[3:4], v122 offset:11
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:9
-; ALIGNED-NEXT: flat_store_byte v[97:98], v105 offset:8
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1384 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[83:84], v105 offset:8
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1376 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[97:98], v0 offset:2
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1364 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0 offset:2
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1352 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[97:98], v0 offset:1
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0 offset:1
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1388 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[97:98], v0 offset:4
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1352 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0 offset:4
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1348 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[97:98], v0
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1344 ; 4-byte Folded Reload
-; ALIGNED-NEXT: v_add_co_u32 v97, vcc_lo, 0x100, v97
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v98, null, 0, v98, vcc_lo
+; ALIGNED-NEXT: flat_store_byte v[83:84], v0
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1340 ; 4-byte Folded Reload
+; ALIGNED-NEXT: v_add_co_u32 v83, vcc_lo, 0x100, v83
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v84, null, 0, v84, vcc_lo
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1380 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1372 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:4
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1368 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1360 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:2
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1372 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1364 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:3
; ALIGNED-NEXT: s_cbranch_scc0 .LBB9_2
@@ -14271,8 +14270,8 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_andn2_saveexec_b32 s6, s6
; ALIGNED-NEXT: s_cbranch_execz .LBB9_6
; ALIGNED-NEXT: ; %bb.4: ; %memmove_bwd_loop.preheader
-; ALIGNED-NEXT: v_add_co_u32 v113, vcc_lo, 0x700, v97
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v114, null, 0, v98, vcc_lo
+; ALIGNED-NEXT: v_add_co_u32 v103, vcc_lo, 0x700, v83
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v104, null, 0, v84, vcc_lo
; ALIGNED-NEXT: s_movk_i32 s4, 0xf800
; ALIGNED-NEXT: s_mov_b32 s5, -1
; ALIGNED-NEXT: .LBB9_5: ; %memmove_bwd_loop
@@ -14285,43 +14284,43 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v7, v2, s[0:3], 0 offen offset:1816
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1817
; ALIGNED-NEXT: buffer_load_ubyte v12, v2, s[0:3], 0 offen offset:1818
-; ALIGNED-NEXT: buffer_load_ubyte v116, v2, s[0:3], 0 offen offset:1811
+; ALIGNED-NEXT: buffer_load_ubyte v125, v2, s[0:3], 0 offen offset:1811
; ALIGNED-NEXT: buffer_load_ubyte v13, v2, s[0:3], 0 offen offset:1819
; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1820
; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:1821
; ALIGNED-NEXT: buffer_load_ubyte v10, v2, s[0:3], 0 offen offset:1822
; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:1823
-; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:1824
+; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:1824
; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:1825
-; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:1826
-; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:1827
-; ALIGNED-NEXT: buffer_load_ubyte v14, v2, s[0:3], 0 offen offset:1828
-; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:1829
-; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:1830
+; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:1826
+; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:1827
+; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:1828
+; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:1829
+; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:1830
; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:1831
-; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:1832
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:1833
-; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:1834
-; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:1835
-; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:1836
-; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:1837
-; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:1838
-; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:1839
+; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:1832
+; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:1833
+; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:1834
+; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:1836
+; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:1837
+; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:1838
+; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:1839
; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:1840
; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:1841
; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:1842
+; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:1835
; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:1843
; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:1844
-; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:1845
-; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:1846
-; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:1847
+; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:1845
+; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:1846
+; ALIGNED-NEXT: buffer_load_ubyte v38, v2, s[0:3], 0 offen offset:1847
; ALIGNED-NEXT: buffer_load_ubyte v48, v2, s[0:3], 0 offen offset:1848
; ALIGNED-NEXT: buffer_load_ubyte v51, v2, s[0:3], 0 offen offset:1849
; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:1850
; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:1852
-; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:1853
+; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:1853
; ALIGNED-NEXT: buffer_load_ubyte v39, v2, s[0:3], 0 offen offset:1854
-; ALIGNED-NEXT: buffer_load_ubyte v49, v2, s[0:3], 0 offen offset:1855
+; ALIGNED-NEXT: buffer_load_ubyte v50, v2, s[0:3], 0 offen offset:1855
; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:1856
; ALIGNED-NEXT: buffer_load_ubyte v55, v2, s[0:3], 0 offen offset:1857
; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:1858
@@ -14364,72 +14363,70 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(47)
; ALIGNED-NEXT: v_lshl_or_b32 v6, v11, 8, v10
; ALIGNED-NEXT: v_lshl_or_b32 v8, v13, 8, v12
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:732 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(45)
-; ALIGNED-NEXT: v_lshl_or_b32 v9, v16, 8, v15
+; ALIGNED-NEXT: v_lshl_or_b32 v9, v16, 8, v14
; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 16, v5
; ALIGNED-NEXT: s_waitcnt vmcnt(43)
-; ALIGNED-NEXT: v_lshl_or_b32 v10, v20, 8, v18
+; ALIGNED-NEXT: v_lshl_or_b32 v10, v18, 8, v17
; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:748 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:736 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(41)
-; ALIGNED-NEXT: v_lshl_or_b32 v11, v17, 8, v14
+; ALIGNED-NEXT: v_lshl_or_b32 v11, v19, 8, v15
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v8, 16, v7
; ALIGNED-NEXT: s_waitcnt vmcnt(39)
-; ALIGNED-NEXT: v_lshl_or_b32 v12, v22, 8, v19
+; ALIGNED-NEXT: v_lshl_or_b32 v12, v22, 8, v20
; ALIGNED-NEXT: buffer_store_dword v13, off, s[0:3], s32 offset:744 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:752 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(37)
+; ALIGNED-NEXT: s_waitcnt vmcnt(34)
; ALIGNED-NEXT: v_lshl_or_b32 v13, v24, 8, v23
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v10, 16, v9
-; ALIGNED-NEXT: s_waitcnt vmcnt(35)
-; ALIGNED-NEXT: v_lshl_or_b32 v14, v30, 8, v26
-; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:760 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(33)
-; ALIGNED-NEXT: v_lshl_or_b32 v15, v27, 8, v25
+; ALIGNED-NEXT: s_waitcnt vmcnt(32)
+; ALIGNED-NEXT: v_lshl_or_b32 v14, v26, 8, v25
+; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:764 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v15, v28, 8, v27
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:832 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v12, 16, v11
-; ALIGNED-NEXT: s_waitcnt vmcnt(31)
-; ALIGNED-NEXT: v_lshl_or_b32 v16, v29, 8, v28
-; ALIGNED-NEXT: s_waitcnt vmcnt(29)
+; ALIGNED-NEXT: s_waitcnt vmcnt(28)
+; ALIGNED-NEXT: v_lshl_or_b32 v16, v29, 8, v30
; ALIGNED-NEXT: v_lshl_or_b32 v3, v32, 8, v31
; ALIGNED-NEXT: s_waitcnt vmcnt(27)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v35, 8, v34
; ALIGNED-NEXT: s_waitcnt vmcnt(25)
-; ALIGNED-NEXT: v_lshl_or_b32 v5, v38, 8, v33
+; ALIGNED-NEXT: v_lshl_or_b32 v5, v36, 8, v33
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v14, 16, v13
; ALIGNED-NEXT: s_waitcnt vmcnt(23)
-; ALIGNED-NEXT: v_lshl_or_b32 v6, v37, 8, v36
+; ALIGNED-NEXT: v_lshl_or_b32 v6, v38, 8, v37
; ALIGNED-NEXT: s_waitcnt vmcnt(18)
-; ALIGNED-NEXT: v_lshl_or_b32 v7, v50, 8, v21
+; ALIGNED-NEXT: v_lshl_or_b32 v7, v49, 8, v21
; ALIGNED-NEXT: s_waitcnt vmcnt(16)
-; ALIGNED-NEXT: v_lshl_or_b32 v8, v49, 8, v39
+; ALIGNED-NEXT: v_lshl_or_b32 v8, v50, 8, v39
; ALIGNED-NEXT: v_lshl_or_b32 v9, v51, 8, v48
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v16, 16, v15
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:708 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:756 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:788 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v18, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(12)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v53, 8, v52
; ALIGNED-NEXT: v_lshl_or_b32 v4, v55, 8, v54
-; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:776 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v6, 16, v5
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
; ALIGNED-NEXT: v_lshl_or_b32 v5, v65, 8, v64
-; ALIGNED-NEXT: buffer_store_dword v23, off, s[0:3], s32 offset:804 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:808 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v26, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v19, off, s[0:3], s32 offset:768 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v20, off, s[0:3], s32 offset:772 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v8, 16, v7
; ALIGNED-NEXT: s_clause 0x4
@@ -14443,37 +14440,37 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(14)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v67, 8, v66
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:1882
-; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:796 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v22, off, s[0:3], s32 offset:780 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:924 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v5, 16, v4
; ALIGNED-NEXT: s_waitcnt vmcnt(13)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v69, 8, v68
; ALIGNED-NEXT: buffer_load_ubyte v5, v2, s[0:3], 0 offen offset:1873
-; ALIGNED-NEXT: buffer_store_dword v27, off, s[0:3], s32 offset:800 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:812 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v28, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v30, off, s[0:3], s32 offset:824 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:956 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:816 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v29, off, s[0:3], s32 offset:820 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v31, off, s[0:3], s32 offset:828 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v32, off, s[0:3], s32 offset:840 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:960 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1871
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v83, 8, v82
-; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v35, off, s[0:3], s32 offset:856 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:844 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:852 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:872 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v36, off, s[0:3], s32 offset:860 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v37, off, s[0:3], s32 offset:864 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v21, off, s[0:3], s32 offset:876 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v49, off, s[0:3], s32 offset:888 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v39, off, s[0:3], s32 offset:880 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v50, off, s[0:3], s32 offset:892 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v48, off, s[0:3], s32 offset:884 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:896 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:904 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v52, off, s[0:3], s32 offset:908 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v53, off, s[0:3], s32 offset:900 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v54, off, s[0:3], s32 offset:920 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v55, off, s[0:3], s32 offset:928 ; 4-byte Folded Spill
@@ -14491,7 +14488,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v83, off, s[0:3], s32 offset:968 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(8)
; ALIGNED-NEXT: buffer_store_dword v84, off, s[0:3], s32 offset:972 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v116, off, s[0:3], s32 offset:1464 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v125, off, s[0:3], s32 offset:1464 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x3
; ALIGNED-NEXT: buffer_load_ubyte v8, v2, s[0:3], 0 offen offset:1883
; ALIGNED-NEXT: buffer_load_ubyte v11, v2, s[0:3], 0 offen offset:1904
@@ -14738,22 +14735,22 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1280 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x3e
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1933
-; ALIGNED-NEXT: buffer_load_ubyte v104, v2, s[0:3], 0 offen offset:1957
+; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:1957
; ALIGNED-NEXT: buffer_load_ubyte v95, v2, s[0:3], 0 offen offset:1958
; ALIGNED-NEXT: buffer_load_ubyte v93, v2, s[0:3], 0 offen offset:1959
; ALIGNED-NEXT: buffer_load_ubyte v91, v2, s[0:3], 0 offen offset:1960
-; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:1961
-; ALIGNED-NEXT: buffer_load_ubyte v58, v2, s[0:3], 0 offen offset:1962
-; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:1963
+; ALIGNED-NEXT: buffer_load_ubyte v75, v2, s[0:3], 0 offen offset:1961
+; ALIGNED-NEXT: buffer_load_ubyte v63, v2, s[0:3], 0 offen offset:1962
+; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:1963
; ALIGNED-NEXT: buffer_load_ubyte v56, v2, s[0:3], 0 offen offset:1964
-; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:1965
+; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:1965
; ALIGNED-NEXT: buffer_load_ubyte v43, v2, s[0:3], 0 offen offset:1966
-; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:1967
+; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:1967
; ALIGNED-NEXT: buffer_load_ubyte v102, v2, s[0:3], 0 offen offset:1968
-; ALIGNED-NEXT: buffer_load_ubyte v100, v2, s[0:3], 0 offen offset:1969
+; ALIGNED-NEXT: buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:1969
; ALIGNED-NEXT: buffer_load_ubyte v97, v2, s[0:3], 0 offen offset:1970
-; ALIGNED-NEXT: buffer_load_ubyte v98, v2, s[0:3], 0 offen offset:1971
-; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:1972
+; ALIGNED-NEXT: buffer_load_ubyte v96, v2, s[0:3], 0 offen offset:1971
+; ALIGNED-NEXT: buffer_load_ubyte v99, v2, s[0:3], 0 offen offset:1972
; ALIGNED-NEXT: buffer_load_ubyte v83, v2, s[0:3], 0 offen offset:1973
; ALIGNED-NEXT: buffer_load_ubyte v84, v2, s[0:3], 0 offen offset:1974
; ALIGNED-NEXT: buffer_load_ubyte v82, v2, s[0:3], 0 offen offset:1975
@@ -14763,9 +14760,9 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v70, v2, s[0:3], 0 offen offset:1979
; ALIGNED-NEXT: buffer_load_ubyte v69, v2, s[0:3], 0 offen offset:1980
; ALIGNED-NEXT: buffer_load_ubyte v67, v2, s[0:3], 0 offen offset:1981
-; ALIGNED-NEXT: buffer_load_ubyte v68, v2, s[0:3], 0 offen offset:1982
-; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:1983
-; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:1984
+; ALIGNED-NEXT: buffer_load_ubyte v66, v2, s[0:3], 0 offen offset:1982
+; ALIGNED-NEXT: buffer_load_ubyte v65, v2, s[0:3], 0 offen offset:1983
+; ALIGNED-NEXT: buffer_load_ubyte v64, v2, s[0:3], 0 offen offset:1984
; ALIGNED-NEXT: buffer_load_ubyte v54, v2, s[0:3], 0 offen offset:1985
; ALIGNED-NEXT: buffer_load_ubyte v53, v2, s[0:3], 0 offen offset:1986
; ALIGNED-NEXT: buffer_load_ubyte v52, v2, s[0:3], 0 offen offset:1987
@@ -14777,55 +14774,58 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v37, v2, s[0:3], 0 offen offset:1993
; ALIGNED-NEXT: buffer_load_ubyte v35, v2, s[0:3], 0 offen offset:1994
; ALIGNED-NEXT: buffer_load_ubyte v36, v2, s[0:3], 0 offen offset:1995
-; ALIGNED-NEXT: buffer_load_ubyte v34, v2, s[0:3], 0 offen offset:1996
+; ALIGNED-NEXT: buffer_load_ubyte v33, v2, s[0:3], 0 offen offset:1996
; ALIGNED-NEXT: buffer_load_ubyte v31, v2, s[0:3], 0 offen offset:1997
; ALIGNED-NEXT: buffer_load_ubyte v32, v2, s[0:3], 0 offen offset:1998
; ALIGNED-NEXT: buffer_load_ubyte v30, v2, s[0:3], 0 offen offset:1999
; ALIGNED-NEXT: buffer_load_ubyte v29, v2, s[0:3], 0 offen offset:2000
; ALIGNED-NEXT: buffer_load_ubyte v27, v2, s[0:3], 0 offen offset:2001
-; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:2002
+; ALIGNED-NEXT: buffer_load_ubyte v25, v2, s[0:3], 0 offen offset:2002
; ALIGNED-NEXT: buffer_load_ubyte v26, v2, s[0:3], 0 offen offset:2003
-; ALIGNED-NEXT: buffer_load_ubyte v24, v2, s[0:3], 0 offen offset:2004
-; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:2005
-; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:2006
-; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:2007
-; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:2008
-; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:2009
+; ALIGNED-NEXT: buffer_load_ubyte v28, v2, s[0:3], 0 offen offset:2004
+; ALIGNED-NEXT: buffer_load_ubyte v23, v2, s[0:3], 0 offen offset:2005
+; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:2006
+; ALIGNED-NEXT: buffer_load_ubyte v21, v2, s[0:3], 0 offen offset:2007
+; ALIGNED-NEXT: buffer_load_ubyte v20, v2, s[0:3], 0 offen offset:2008
+; ALIGNED-NEXT: buffer_load_ubyte v19, v2, s[0:3], 0 offen offset:2009
; ALIGNED-NEXT: buffer_load_ubyte v18, v2, s[0:3], 0 offen offset:2010
-; ALIGNED-NEXT: buffer_load_ubyte v16, v2, s[0:3], 0 offen offset:2011
-; ALIGNED-NEXT: buffer_load_ubyte v22, v2, s[0:3], 0 offen offset:2012
+; ALIGNED-NEXT: buffer_load_ubyte v17, v2, s[0:3], 0 offen offset:2011
+; ALIGNED-NEXT: buffer_load_ubyte v15, v2, s[0:3], 0 offen offset:2012
; ALIGNED-NEXT: buffer_load_ubyte v105, v2, s[0:3], 0 offen offset:2029
; ALIGNED-NEXT: buffer_load_ubyte v106, v2, s[0:3], 0 offen offset:2030
; ALIGNED-NEXT: buffer_load_ubyte v94, v2, s[0:3], 0 offen offset:2031
; ALIGNED-NEXT: buffer_load_ubyte v92, v2, s[0:3], 0 offen offset:2032
; ALIGNED-NEXT: buffer_load_ubyte v90, v2, s[0:3], 0 offen offset:2033
; ALIGNED-NEXT: buffer_load_ubyte v88, v2, s[0:3], 0 offen offset:2034
-; ALIGNED-NEXT: s_clause 0x18
+; ALIGNED-NEXT: s_clause 0x1b
; ALIGNED-NEXT: buffer_load_ubyte v78, v2, s[0:3], 0 offen offset:2035
; ALIGNED-NEXT: buffer_load_ubyte v76, v2, s[0:3], 0 offen offset:2036
; ALIGNED-NEXT: buffer_load_ubyte v74, v2, s[0:3], 0 offen offset:2037
; ALIGNED-NEXT: buffer_load_ubyte v72, v2, s[0:3], 0 offen offset:2038
-; ALIGNED-NEXT: buffer_load_ubyte v61, v2, s[0:3], 0 offen offset:2039
-; ALIGNED-NEXT: buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:2040
-; ALIGNED-NEXT: buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:2041
+; ALIGNED-NEXT: buffer_load_ubyte v62, v2, s[0:3], 0 offen offset:2039
+; ALIGNED-NEXT: buffer_load_ubyte v59, v2, s[0:3], 0 offen offset:2040
+; ALIGNED-NEXT: buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:2041
; ALIGNED-NEXT: buffer_load_ubyte v57, v2, s[0:3], 0 offen offset:2042
; ALIGNED-NEXT: buffer_load_ubyte v46, v2, s[0:3], 0 offen offset:2043
; ALIGNED-NEXT: buffer_load_ubyte v44, v2, s[0:3], 0 offen offset:2044
-; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:2045
-; ALIGNED-NEXT: buffer_load_ubyte v41, v2, s[0:3], 0 offen offset:2046
-; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:2047
+; ALIGNED-NEXT: buffer_load_ubyte v42, v2, s[0:3], 0 offen offset:2045
+; ALIGNED-NEXT: buffer_load_ubyte v40, v2, s[0:3], 0 offen offset:2046
+; ALIGNED-NEXT: buffer_load_ubyte v118, v2, s[0:3], 0 offen offset:2047
; ALIGNED-NEXT: buffer_load_ubyte v121, v2, s[0:3], 0 offen offset:1793
-; ALIGNED-NEXT: buffer_load_ubyte v107, v2, s[0:3], 0 offen offset:1794
-; ALIGNED-NEXT: buffer_load_ubyte v99, v2, s[0:3], 0 offen offset:1795
-; ALIGNED-NEXT: buffer_load_ubyte v109, v2, s[0:3], 0 offen offset:1796
+; ALIGNED-NEXT: buffer_load_ubyte v113, v2, s[0:3], 0 offen offset:1794
+; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:1795
+; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:1796
; ALIGNED-NEXT: buffer_load_ubyte v127, v2, s[0:3], 0 offen offset:1797
-; ALIGNED-NEXT: buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:1798
+; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:1798
; ALIGNED-NEXT: buffer_load_ubyte v89, v2, s[0:3], 0 offen offset:1804
-; ALIGNED-NEXT: buffer_load_ubyte v60, v2, s[0:3], 0 offen offset:1805
+; ALIGNED-NEXT: buffer_load_ubyte v79, v2, s[0:3], 0 offen offset:1805
; ALIGNED-NEXT: buffer_load_ubyte v77, v2, s[0:3], 0 offen offset:1806
; ALIGNED-NEXT: buffer_load_ubyte v73, v2, s[0:3], 0 offen offset:1807
-; ALIGNED-NEXT: buffer_load_ubyte v117, v2, s[0:3], 0 offen offset:1803
-; ALIGNED-NEXT: buffer_load_ubyte v111, v2, s[0:3], 0 offen offset:1799
+; ALIGNED-NEXT: buffer_load_ubyte v45, v2, s[0:3], 0 offen offset:1803
+; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:1800
+; ALIGNED-NEXT: buffer_load_ubyte v119, v2, s[0:3], 0 offen offset:1801
+; ALIGNED-NEXT: buffer_load_ubyte v115, v2, s[0:3], 0 offen offset:1802
+; ALIGNED-NEXT: buffer_load_ubyte v123, v2, s[0:3], 0 offen offset:1799
; ALIGNED-NEXT: s_waitcnt vmcnt(62)
; ALIGNED-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:1308 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v9, off, s[0:3], s32 offset:1316 ; 4-byte Folded Spill
@@ -14840,8 +14840,8 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v6
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1288 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v6, v2, s[0:3], 0 offen offset:1948
-; ALIGNED-NEXT: s_waitcnt vmcnt(22)
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v61, 8, v72
+; ALIGNED-NEXT: s_waitcnt vmcnt(25)
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v62, 8, v72
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v8, 8, v7
; ALIGNED-NEXT: v_lshl_or_b32 v4, v10, 8, v9
@@ -14868,11 +14868,12 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:1344 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1340 ; 4-byte Folded Spill
; ALIGNED-NEXT: buffer_load_ubyte v0, v2, s[0:3], 0 offen offset:1941
+; ALIGNED-NEXT: s_waitcnt vmcnt(27)
+; ALIGNED-NEXT: v_lshl_or_b32 v87, v118, 8, v40
; ALIGNED-NEXT: s_waitcnt vmcnt(24)
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v42, 8, v41
-; ALIGNED-NEXT: s_waitcnt vmcnt(18)
-; ALIGNED-NEXT: buffer_store_dword v103, off, s[0:3], s32 offset:1420 ; 4-byte Folded Spill
-; ALIGNED-NEXT: buffer_store_dword v99, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v101, off, s[0:3], s32 offset:1404 ; 4-byte Folded Spill
+; ALIGNED-NEXT: s_waitcnt vmcnt(21)
+; ALIGNED-NEXT: buffer_store_dword v112, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
; ALIGNED-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:1352 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(10)
@@ -14886,9 +14887,9 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: s_waitcnt vmcnt(6)
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:1392 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
-; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1416 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v12, off, s[0:3], s32 offset:1420 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(4)
-; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1404 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v11, off, s[0:3], s32 offset:1408 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
; ALIGNED-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:1424 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_waitcnt vmcnt(2)
@@ -14932,35 +14933,35 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v9, v2, s[0:3], 0 offen offset:2020
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1400 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v104, 8, v5
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v107, 8, v5
; ALIGNED-NEXT: v_lshl_or_b32 v4, v93, 8, v95
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1432 ; 4-byte Folded Spill
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1436 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v45, 8, v56
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v119, 8, v43
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v41, 8, v56
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v117, 8, v43
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v79, 8, v91
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v63, 8, v58
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v75, 8, v91
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v61, 8, v63
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1460 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v100, 8, v102
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v98, 8, v97
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v98, 8, v102
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v96, 8, v97
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1468 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v83, 8, v101
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v83, 8, v99
; ALIGNED-NEXT: v_lshl_or_b32 v4, v82, 8, v84
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1472 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v67, 8, v69
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v66, 8, v68
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v65, 8, v66
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1476 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v80, 8, v81
; ALIGNED-NEXT: v_lshl_or_b32 v4, v70, 8, v71
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Spill
; ALIGNED-NEXT: v_lshl_or_b32 v0, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v54, 8, v65
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v54, 8, v64
; ALIGNED-NEXT: v_lshl_or_b32 v4, v52, 8, v53
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x7
@@ -14972,40 +14973,40 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_ubyte v120, v2, s[0:3], 0 offen offset:2026
; ALIGNED-NEXT: buffer_load_ubyte v110, v2, s[0:3], 0 offen offset:2027
; ALIGNED-NEXT: buffer_load_ubyte v108, v2, s[0:3], 0 offen offset:2028
-; ALIGNED-NEXT: v_lshl_or_b32 v123, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v109, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v49, 8, v55
; ALIGNED-NEXT: v_lshl_or_b32 v4, v48, 8, v50
-; ALIGNED-NEXT: v_lshl_or_b32 v75, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v31, 8, v34
+; ALIGNED-NEXT: v_lshl_or_b32 v58, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v31, 8, v33
; ALIGNED-NEXT: v_lshl_or_b32 v4, v30, 8, v32
-; ALIGNED-NEXT: v_lshl_or_b32 v86, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v100, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v3, v37, 8, v39
; ALIGNED-NEXT: v_lshl_or_b32 v4, v36, 8, v35
; ALIGNED-NEXT: v_lshl_or_b32 v85, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v19, 8, v24
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v17, 8, v20
-; ALIGNED-NEXT: v_lshl_or_b32 v64, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v21, 8, v23
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v16, 8, v18
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v27, 8, v29
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v26, 8, v25
+; ALIGNED-NEXT: v_lshl_or_b32 v68, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v23, 8, v28
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v21, 8, v22
; ALIGNED-NEXT: v_lshl_or_b32 v51, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(15)
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v13, 8, v22
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v13, 8, v15
; ALIGNED-NEXT: s_waitcnt vmcnt(13)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v12, 8, v14
; ALIGNED-NEXT: v_lshl_or_b32 v38, v4, 16, v3
-; ALIGNED-NEXT: v_lshl_or_b32 v3, v27, 8, v29
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v26, 8, v28
-; ALIGNED-NEXT: v_lshl_or_b32 v33, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v3, v19, 8, v20
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v17, 8, v18
+; ALIGNED-NEXT: v_lshl_or_b32 v34, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(11)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v8, 8, v11
; ALIGNED-NEXT: s_waitcnt vmcnt(9)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v6, 8, v7
-; ALIGNED-NEXT: v_lshl_or_b32 v25, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v24, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(7)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v0, 8, v9
; ALIGNED-NEXT: s_waitcnt vmcnt(5)
; ALIGNED-NEXT: v_lshl_or_b32 v4, v126, 8, v1
-; ALIGNED-NEXT: v_lshl_or_b32 v15, v4, 16, v3
+; ALIGNED-NEXT: v_lshl_or_b32 v16, v4, 16, v3
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: v_lshl_or_b32 v3, v105, 8, v108
; ALIGNED-NEXT: v_lshl_or_b32 v4, v94, 8, v106
@@ -15017,67 +15018,61 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: v_lshl_or_b32 v4, v78, 8, v88
; ALIGNED-NEXT: v_lshl_or_b32 v3, v4, 16, v3
; ALIGNED-NEXT: v_lshl_or_b32 v4, v74, 8, v76
-; ALIGNED-NEXT: v_lshl_or_b32 v4, v87, 16, v4
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v40, 8, v44
-; ALIGNED-NEXT: v_lshl_or_b32 v118, v96, 16, v87
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v59, 8, v62
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v46, 8, v57
-; ALIGNED-NEXT: v_lshl_or_b32 v115, v96, 16, v87
-; ALIGNED-NEXT: buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:1792
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v99, 8, v107
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v87, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v121, 8, v87
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v96, 16, v87
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v111, 8, v103
-; ALIGNED-NEXT: s_clause 0x2
-; ALIGNED-NEXT: buffer_load_ubyte v47, v2, s[0:3], 0 offen offset:1800
-; ALIGNED-NEXT: buffer_load_ubyte v112, v2, s[0:3], 0 offen offset:1801
-; ALIGNED-NEXT: buffer_load_ubyte v103, v2, s[0:3], 0 offen offset:1802
-; ALIGNED-NEXT: buffer_store_dword v87, off, s[0:3], s32 offset:1436 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v127, 8, v109
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v96, 16, v87
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v73, 8, v77
-; ALIGNED-NEXT: buffer_store_dword v87, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v60, 8, v89
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v96, 16, v87
-; ALIGNED-NEXT: buffer_store_dword v87, off, s[0:3], s32 offset:1452 ; 4-byte Folded Spill
-; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v112, 8, v47
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: v_lshl_or_b32 v96, v117, 8, v103
-; ALIGNED-NEXT: v_lshl_or_b32 v87, v96, 16, v87
-; ALIGNED-NEXT: buffer_store_dword v87, off, s[0:3], s32 offset:1456 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v4, v86, 16, v4
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v42, 8, v44
+; ALIGNED-NEXT: v_lshl_or_b32 v116, v87, 16, v86
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v60, 8, v59
+; ALIGNED-NEXT: v_lshl_or_b32 v87, v46, 8, v57
+; ALIGNED-NEXT: v_lshl_or_b32 v114, v87, 16, v86
+; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:1792
+; ALIGNED-NEXT: v_lshl_or_b32 v87, v101, 8, v113
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1412 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v121, 8, v86
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v87, 16, v86
+; ALIGNED-NEXT: v_lshl_or_b32 v87, v123, 8, v112
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1432 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v127, 8, v111
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v87, 16, v86
+; ALIGNED-NEXT: v_lshl_or_b32 v87, v73, 8, v77
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1444 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v79, 8, v89
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v87, 16, v86
+; ALIGNED-NEXT: v_lshl_or_b32 v87, v45, 8, v115
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1452 ; 4-byte Folded Spill
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v119, 8, v47
+; ALIGNED-NEXT: v_lshl_or_b32 v86, v87, 16, v86
+; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:1456 ; 4-byte Folded Spill
; ALIGNED-NEXT: s_clause 0x2
-; ALIGNED-NEXT: buffer_load_ubyte v96, v2, s[0:3], 0 offen offset:1810
-; ALIGNED-NEXT: buffer_load_ubyte v99, v2, s[0:3], 0 offen offset:1808
+; ALIGNED-NEXT: buffer_load_ubyte v86, v2, s[0:3], 0 offen offset:1810
+; ALIGNED-NEXT: buffer_load_ubyte v101, v2, s[0:3], 0 offen offset:1808
; ALIGNED-NEXT: buffer_load_ubyte v87, v2, s[0:3], 0 offen offset:1809
-; ALIGNED-NEXT: buffer_store_dword v115, off, s[0:3], s32 offset:488
-; ALIGNED-NEXT: buffer_store_dword v118, off, s[0:3], s32 offset:492
+; ALIGNED-NEXT: buffer_store_dword v114, off, s[0:3], s32 offset:488
+; ALIGNED-NEXT: buffer_store_dword v116, off, s[0:3], s32 offset:492
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:484
; ALIGNED-NEXT: buffer_store_dword v3, off, s[0:3], s32 offset:480
-; ALIGNED-NEXT: v_add_co_u32 v3, vcc_lo, v113, 3
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v114, vcc_lo
+; ALIGNED-NEXT: v_add_co_u32 v3, vcc_lo, v103, 3
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v4, null, 0, v104, vcc_lo
; ALIGNED-NEXT: flat_store_byte v[3:4], v57 offset:247
; ALIGNED-NEXT: flat_store_byte v[3:4], v46 offset:248
-; ALIGNED-NEXT: flat_store_byte v[3:4], v59 offset:246
-; ALIGNED-NEXT: flat_store_byte v[3:4], v42 offset:252
-; ALIGNED-NEXT: flat_store_byte v[3:4], v40 offset:250
-; ALIGNED-NEXT: flat_store_byte v[3:4], v41 offset:251
+; ALIGNED-NEXT: flat_store_byte v[3:4], v60 offset:246
+; ALIGNED-NEXT: flat_store_byte v[3:4], v118 offset:252
+; ALIGNED-NEXT: flat_store_byte v[3:4], v42 offset:250
+; ALIGNED-NEXT: flat_store_byte v[3:4], v40 offset:251
; ALIGNED-NEXT: flat_store_byte v[3:4], v44 offset:249
-; ALIGNED-NEXT: flat_store_byte v[3:4], v62 offset:245
+; ALIGNED-NEXT: flat_store_byte v[3:4], v59 offset:245
; ALIGNED-NEXT: flat_store_byte v[3:4], v88 offset:239
; ALIGNED-NEXT: flat_store_byte v[3:4], v78 offset:240
; ALIGNED-NEXT: flat_store_byte v[3:4], v90 offset:238
-; ALIGNED-NEXT: flat_store_byte v[3:4], v61 offset:244
+; ALIGNED-NEXT: flat_store_byte v[3:4], v62 offset:244
; ALIGNED-NEXT: flat_store_byte v[3:4], v74 offset:242
; ALIGNED-NEXT: flat_store_byte v[3:4], v72 offset:243
; ALIGNED-NEXT: flat_store_byte v[3:4], v76 offset:241
; ALIGNED-NEXT: flat_store_byte v[3:4], v92 offset:237
; ALIGNED-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:504
; ALIGNED-NEXT: buffer_store_dword v10, off, s[0:3], s32 offset:508
-; ALIGNED-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:500
-; ALIGNED-NEXT: buffer_store_dword v25, off, s[0:3], s32 offset:496
+; ALIGNED-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:500
+; ALIGNED-NEXT: buffer_store_dword v24, off, s[0:3], s32 offset:496
; ALIGNED-NEXT: flat_store_byte v[3:4], v120 offset:231
; ALIGNED-NEXT: flat_store_byte v[3:4], v110 offset:232
; ALIGNED-NEXT: flat_store_byte v[3:4], v122 offset:230
@@ -15094,30 +15089,30 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v1 offset:227
; ALIGNED-NEXT: flat_store_byte v[3:4], v9 offset:225
; ALIGNED-NEXT: flat_store_byte v[3:4], v11 offset:221
-; ALIGNED-NEXT: buffer_store_dword v33, off, s[0:3], s32 offset:448
+; ALIGNED-NEXT: buffer_store_dword v34, off, s[0:3], s32 offset:456
; ALIGNED-NEXT: buffer_store_dword v38, off, s[0:3], s32 offset:460
-; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:456
-; ALIGNED-NEXT: buffer_store_dword v64, off, s[0:3], s32 offset:452
-; ALIGNED-NEXT: flat_store_byte v[3:4], v27 offset:206
-; ALIGNED-NEXT: flat_store_byte v[3:4], v26 offset:208
-; ALIGNED-NEXT: flat_store_byte v[3:4], v28 offset:207
-; ALIGNED-NEXT: flat_store_byte v[3:4], v19 offset:210
-; ALIGNED-NEXT: flat_store_byte v[3:4], v17 offset:212
-; ALIGNED-NEXT: flat_store_byte v[3:4], v20 offset:211
-; ALIGNED-NEXT: flat_store_byte v[3:4], v24 offset:209
+; ALIGNED-NEXT: buffer_store_dword v51, off, s[0:3], s32 offset:452
+; ALIGNED-NEXT: buffer_store_dword v68, off, s[0:3], s32 offset:448
; ALIGNED-NEXT: flat_store_byte v[3:4], v18 offset:215
-; ALIGNED-NEXT: flat_store_byte v[3:4], v16 offset:216
-; ALIGNED-NEXT: flat_store_byte v[3:4], v21 offset:214
+; ALIGNED-NEXT: flat_store_byte v[3:4], v17 offset:216
+; ALIGNED-NEXT: flat_store_byte v[3:4], v19 offset:214
; ALIGNED-NEXT: flat_store_byte v[3:4], v12 offset:220
; ALIGNED-NEXT: flat_store_byte v[3:4], v13 offset:218
; ALIGNED-NEXT: flat_store_byte v[3:4], v14 offset:219
-; ALIGNED-NEXT: flat_store_byte v[3:4], v22 offset:217
-; ALIGNED-NEXT: flat_store_byte v[3:4], v23 offset:213
+; ALIGNED-NEXT: flat_store_byte v[3:4], v15 offset:217
+; ALIGNED-NEXT: flat_store_byte v[3:4], v20 offset:213
+; ALIGNED-NEXT: flat_store_byte v[3:4], v25 offset:207
+; ALIGNED-NEXT: flat_store_byte v[3:4], v26 offset:208
+; ALIGNED-NEXT: flat_store_byte v[3:4], v27 offset:206
+; ALIGNED-NEXT: flat_store_byte v[3:4], v21 offset:212
+; ALIGNED-NEXT: flat_store_byte v[3:4], v23 offset:210
+; ALIGNED-NEXT: flat_store_byte v[3:4], v22 offset:211
+; ALIGNED-NEXT: flat_store_byte v[3:4], v28 offset:209
; ALIGNED-NEXT: flat_store_byte v[3:4], v29 offset:205
; ALIGNED-NEXT: buffer_store_dword v85, off, s[0:3], s32 offset:472
-; ALIGNED-NEXT: buffer_store_dword v86, off, s[0:3], s32 offset:476
-; ALIGNED-NEXT: buffer_store_dword v75, off, s[0:3], s32 offset:468
-; ALIGNED-NEXT: buffer_store_dword v123, off, s[0:3], s32 offset:464
+; ALIGNED-NEXT: buffer_store_dword v100, off, s[0:3], s32 offset:476
+; ALIGNED-NEXT: buffer_store_dword v58, off, s[0:3], s32 offset:468
+; ALIGNED-NEXT: buffer_store_dword v109, off, s[0:3], s32 offset:464
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1484 ; 4-byte Folded Reload
; ALIGNED-NEXT: flat_store_byte v[3:4], v35 offset:199
; ALIGNED-NEXT: flat_store_byte v[3:4], v36 offset:200
@@ -15125,7 +15120,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v30 offset:204
; ALIGNED-NEXT: flat_store_byte v[3:4], v31 offset:202
; ALIGNED-NEXT: flat_store_byte v[3:4], v32 offset:203
-; ALIGNED-NEXT: flat_store_byte v[3:4], v34 offset:201
+; ALIGNED-NEXT: flat_store_byte v[3:4], v33 offset:201
; ALIGNED-NEXT: flat_store_byte v[3:4], v39 offset:197
; ALIGNED-NEXT: flat_store_byte v[3:4], v53 offset:191
; ALIGNED-NEXT: flat_store_byte v[3:4], v52 offset:192
@@ -15134,13 +15129,13 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v49 offset:194
; ALIGNED-NEXT: flat_store_byte v[3:4], v50 offset:195
; ALIGNED-NEXT: flat_store_byte v[3:4], v55 offset:193
-; ALIGNED-NEXT: flat_store_byte v[3:4], v65 offset:189
+; ALIGNED-NEXT: flat_store_byte v[3:4], v64 offset:189
; ALIGNED-NEXT: v_add_nc_u32_e32 v2, 0xffffff00, v2
; ALIGNED-NEXT: s_waitcnt vmcnt(3)
-; ALIGNED-NEXT: v_lshl_or_b32 v116, v116, 8, v96
+; ALIGNED-NEXT: v_lshl_or_b32 v112, v125, 8, v86
; ALIGNED-NEXT: s_waitcnt vmcnt(1)
-; ALIGNED-NEXT: v_lshl_or_b32 v125, v87, 8, v99
-; ALIGNED-NEXT: v_lshl_or_b32 v116, v116, 16, v125
+; ALIGNED-NEXT: v_lshl_or_b32 v125, v87, 8, v101
+; ALIGNED-NEXT: v_lshl_or_b32 v112, v112, 16, v125
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:552
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1480 ; 4-byte Folded Reload
@@ -15156,18 +15151,18 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v71 offset:183
; ALIGNED-NEXT: flat_store_byte v[3:4], v70 offset:184
; ALIGNED-NEXT: flat_store_byte v[3:4], v80 offset:182
-; ALIGNED-NEXT: flat_store_byte v[3:4], v66 offset:188
+; ALIGNED-NEXT: flat_store_byte v[3:4], v65 offset:188
; ALIGNED-NEXT: flat_store_byte v[3:4], v67 offset:186
-; ALIGNED-NEXT: flat_store_byte v[3:4], v68 offset:187
+; ALIGNED-NEXT: flat_store_byte v[3:4], v66 offset:187
; ALIGNED-NEXT: flat_store_byte v[3:4], v69 offset:185
; ALIGNED-NEXT: flat_store_byte v[3:4], v81 offset:181
; ALIGNED-NEXT: flat_store_byte v[3:4], v97 offset:175
-; ALIGNED-NEXT: flat_store_byte v[3:4], v98 offset:176
-; ALIGNED-NEXT: flat_store_byte v[3:4], v100 offset:174
+; ALIGNED-NEXT: flat_store_byte v[3:4], v96 offset:176
+; ALIGNED-NEXT: flat_store_byte v[3:4], v98 offset:174
; ALIGNED-NEXT: flat_store_byte v[3:4], v82 offset:180
; ALIGNED-NEXT: flat_store_byte v[3:4], v83 offset:178
; ALIGNED-NEXT: flat_store_byte v[3:4], v84 offset:179
-; ALIGNED-NEXT: flat_store_byte v[3:4], v101 offset:177
+; ALIGNED-NEXT: flat_store_byte v[3:4], v99 offset:177
; ALIGNED-NEXT: flat_store_byte v[3:4], v102 offset:173
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:568
@@ -15177,15 +15172,15 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1448 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:564
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1432 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1436 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:560
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1428 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v58 offset:167
-; ALIGNED-NEXT: flat_store_byte v[3:4], v63 offset:168
-; ALIGNED-NEXT: flat_store_byte v[3:4], v79 offset:166
-; ALIGNED-NEXT: flat_store_byte v[3:4], v119 offset:172
-; ALIGNED-NEXT: flat_store_byte v[3:4], v45 offset:170
+; ALIGNED-NEXT: flat_store_byte v[3:4], v63 offset:167
+; ALIGNED-NEXT: flat_store_byte v[3:4], v61 offset:168
+; ALIGNED-NEXT: flat_store_byte v[3:4], v75 offset:166
+; ALIGNED-NEXT: flat_store_byte v[3:4], v117 offset:172
+; ALIGNED-NEXT: flat_store_byte v[3:4], v41 offset:170
; ALIGNED-NEXT: flat_store_byte v[3:4], v43 offset:171
; ALIGNED-NEXT: flat_store_byte v[3:4], v56 offset:169
; ALIGNED-NEXT: flat_store_byte v[3:4], v91 offset:165
@@ -15194,16 +15189,16 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1424 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:160
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1420 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:158
; ALIGNED-NEXT: flat_store_byte v[3:4], v93 offset:164
-; ALIGNED-NEXT: flat_store_byte v[3:4], v104 offset:162
+; ALIGNED-NEXT: flat_store_byte v[3:4], v107 offset:162
; ALIGNED-NEXT: flat_store_byte v[3:4], v95 offset:163
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1440 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:161
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1404 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:157
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1400 ; 4-byte Folded Reload
@@ -15325,7 +15320,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:129
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1244 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[113:114], v0 offset:128
+; ALIGNED-NEXT: flat_store_byte v[103:104], v0 offset:128
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1240 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:616
@@ -15565,7 +15560,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:65
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:920 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[113:114], v0 offset:64
+; ALIGNED-NEXT: flat_store_byte v[103:104], v0 offset:64
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:924 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:680
@@ -15575,13 +15570,10 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:912 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:676
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:672
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:58
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:904 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:908 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:55
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:900 ; 4-byte Folded Reload
@@ -15590,9 +15582,12 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:896 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:54
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:892 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:60
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:888 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:58
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:880 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:59
@@ -15602,21 +15597,21 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:884 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:53
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:50
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:47
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:856 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:48
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:840 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:46
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:864 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:872 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:52
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:852 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:50
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:860 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:51
@@ -15628,10 +15623,10 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:45
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:868 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:700
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:696
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:848 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:696
+; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:700
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:836 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:692
@@ -15640,62 +15635,62 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:688
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:824 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:40
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:39
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
-; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:38
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:820 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:37
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:40
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:816 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:44
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:38
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:812 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:43
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:44
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:804 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:42
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:808 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:43
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:796 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:41
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:800 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:32
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:37
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:788 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:31
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:30
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:32
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[113:114], v0 offset:32
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:776 ; 4-byte Folded Reload
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:30
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:780 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:36
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:768 ; 4-byte Folded Reload
+; ALIGNED-NEXT: s_waitcnt vmcnt(0)
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:34
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:772 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:35
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:756 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:34
+; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:33
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:752 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:33
+; ALIGNED-NEXT: flat_store_byte v[103:104], v0 offset:32
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:792 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:648
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:784 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:652
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:764 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:760 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:644
-; ALIGNED-NEXT: buffer_store_dword v116, off, s[0:3], s32 offset:640
+; ALIGNED-NEXT: buffer_store_dword v112, off, s[0:3], s32 offset:640
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:748 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:23
@@ -15720,7 +15715,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:724 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:21
-; ALIGNED-NEXT: flat_store_byte v[3:4], v96 offset:15
+; ALIGNED-NEXT: flat_store_byte v[3:4], v86 offset:15
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1464 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:16
@@ -15737,7 +15732,7 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:704 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:17
-; ALIGNED-NEXT: flat_store_byte v[113:114], v99 offset:16
+; ALIGNED-NEXT: flat_store_byte v[103:104], v101 offset:16
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1456 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:664
@@ -15747,31 +15742,31 @@ define void @memmove_p0_p5_sz2048(ptr addrspace(0) align 1 %dst, ptr addrspace(5
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1444 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:660
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1436 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1432 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: buffer_store_dword v0, off, s[0:3], s32 offset:656
; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1412 ; 4-byte Folded Reload
-; ALIGNED-NEXT: flat_store_byte v[3:4], v103 offset:7
-; ALIGNED-NEXT: flat_store_byte v[3:4], v117 offset:8
-; ALIGNED-NEXT: flat_store_byte v[3:4], v60 offset:10
-; ALIGNED-NEXT: flat_store_byte v[3:4], v112 offset:6
+; ALIGNED-NEXT: flat_store_byte v[3:4], v115 offset:7
+; ALIGNED-NEXT: flat_store_byte v[3:4], v45 offset:8
+; ALIGNED-NEXT: flat_store_byte v[3:4], v119 offset:6
; ALIGNED-NEXT: flat_store_byte v[3:4], v73 offset:12
+; ALIGNED-NEXT: flat_store_byte v[3:4], v79 offset:10
; ALIGNED-NEXT: flat_store_byte v[3:4], v77 offset:11
; ALIGNED-NEXT: flat_store_byte v[3:4], v89 offset:9
-; ALIGNED-NEXT: flat_store_byte v[113:114], v47 offset:8
-; ALIGNED-NEXT: flat_store_byte v[113:114], v107 offset:2
-; ALIGNED-NEXT: flat_store_byte v[113:114], v121 offset:1
-; ALIGNED-NEXT: flat_store_byte v[113:114], v109 offset:4
+; ALIGNED-NEXT: flat_store_byte v[103:104], v47 offset:8
+; ALIGNED-NEXT: flat_store_byte v[103:104], v113 offset:2
+; ALIGNED-NEXT: flat_store_byte v[103:104], v121 offset:1
+; ALIGNED-NEXT: flat_store_byte v[103:104], v111 offset:4
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
-; ALIGNED-NEXT: flat_store_byte v[113:114], v0
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1408 ; 4-byte Folded Reload
-; ALIGNED-NEXT: v_add_co_u32 v113, vcc_lo, 0xffffff00, v113
-; ALIGNED-NEXT: v_add_co_ci_u32_e64 v114, null, -1, v114, vcc_lo
+; ALIGNED-NEXT: flat_store_byte v[103:104], v0
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1404 ; 4-byte Folded Reload
+; ALIGNED-NEXT: v_add_co_u32 v103, vcc_lo, 0xffffff00, v103
+; ALIGNED-NEXT: v_add_co_ci_u32_e64 v104, null, -1, v104, vcc_lo
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0
-; ALIGNED-NEXT: flat_store_byte v[3:4], v111 offset:4
+; ALIGNED-NEXT: flat_store_byte v[3:4], v123 offset:4
; ALIGNED-NEXT: flat_store_byte v[3:4], v127 offset:2
-; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1420 ; 4-byte Folded Reload
+; ALIGNED-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:1416 ; 4-byte Folded Reload
; ALIGNED-NEXT: s_waitcnt vmcnt(0)
; ALIGNED-NEXT: flat_store_byte v[3:4], v0 offset:3
; ALIGNED-NEXT: s_cbranch_scc0 .LBB9_5
@@ -16009,30 +16004,30 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v3, null, s7, v1, vcc_lo
; ALIGNED-NEXT: s_add_u32 s6, s6, 0x100
; ALIGNED-NEXT: s_addc_u32 s7, s7, 0
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:204
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:208
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:128
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:12
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:8
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:4
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:64
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:108
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:104
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:32
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:60
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:52
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:16
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:76
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:72
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:68
; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:16
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:8
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:4
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:2
@@ -16041,10 +16036,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: v_add_co_u32 v2, s4, v2, 3
; ALIGNED-NEXT: v_cmp_gt_u64_e64 vcc_lo, 0x800, s[6:7]
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v3, null, 0, v3, s4
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:152
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:156
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:160
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:247
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:248
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:246
@@ -16061,10 +16056,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:239
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:238
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:237
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:168
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:172
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:164
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:176
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:231
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:232
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:230
@@ -16081,10 +16076,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:223
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:222
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:221
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:120
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:124
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:128
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:215
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:216
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:214
@@ -16101,10 +16096,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:207
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:206
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:205
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:136
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:140
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:132
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:144
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:199
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:200
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:198
@@ -16121,10 +16116,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:191
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:190
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:189
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:216
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:220
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:212
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:183
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:184
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:182
@@ -16141,10 +16136,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:175
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:174
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:173
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:236
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:167
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:168
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:166
@@ -16161,10 +16156,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:159
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:158
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:157
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:184
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:188
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:192
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:151
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:152
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:150
@@ -16196,10 +16191,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:128
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:127
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:126
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:24
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:28
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:20
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:32
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:119
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:120
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:118
@@ -16216,10 +16211,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:111
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:110
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:109
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:44
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:36
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:103
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:104
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:102
@@ -16236,26 +16231,26 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:95
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:94
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:93
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:248
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:244
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:240
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:78
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:77
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:80
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:79
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:84
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:83
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:82
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:81
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:86
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:85
-; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:88
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:87
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:88
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:86
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:92
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:91
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:90
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:89
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:85
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:84
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:83
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:82
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:81
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:80
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:79
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:78
+; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:77
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:71
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:72
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:70
@@ -16271,10 +16266,10 @@ define void @memset_p0_sz2048(ptr addrspace(0) %dst) #1 {
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:64
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:63
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:62
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:92
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:96
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:55
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:56
; ALIGNED-NEXT: flat_store_byte v[2:3], v5 offset:54
@@ -16433,10 +16428,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: s_add_u32 s4, s4, 0x100
; ALIGNED-NEXT: v_add_co_ci_u32_e64 v3, null, s5, v1, vcc_lo
; ALIGNED-NEXT: s_addc_u32 s5, s5, 0
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:152
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:156
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:144
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:168
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:172
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:164
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:160
; ALIGNED-NEXT: v_cmp_gt_u64_e64 vcc_lo, 0x800, s[4:5]
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:250
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:251
@@ -16454,10 +16449,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:242
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:241
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:240
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:168
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:172
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:164
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:160
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:184
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:188
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:176
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:234
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:235
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:233
@@ -16474,10 +16469,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:226
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:225
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:224
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:120
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:124
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:136
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:140
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:132
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:128
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:218
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:219
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:217
@@ -16494,10 +16489,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:210
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:209
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:208
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:136
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:140
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:132
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:128
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:152
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:156
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:148
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:144
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:202
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:203
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:201
@@ -16514,10 +16509,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:194
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:193
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:192
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:216
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:220
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:212
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:208
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:236
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:186
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:187
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:185
@@ -16534,10 +16529,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:178
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:177
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:176
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:232
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:236
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:228
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:224
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:248
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:244
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:240
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:170
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:171
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:169
@@ -16554,10 +16549,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:162
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:161
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:160
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:184
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:188
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:180
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:176
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:204
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:192
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:154
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:155
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:153
@@ -16574,10 +16569,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:146
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:145
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:144
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:200
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:204
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:196
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:192
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:216
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:220
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:212
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:208
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:138
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:139
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:137
@@ -16594,10 +16589,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:130
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:129
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:128
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:24
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:28
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:20
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:44
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:36
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:32
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:122
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:123
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:121
@@ -16614,10 +16609,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:114
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:113
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:112
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:40
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:44
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:36
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:32
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:60
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:52
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:106
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:107
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:105
@@ -16634,30 +16629,30 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:98
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:97
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:96
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:252
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:248
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:244
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:240
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:81
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:80
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:83
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:82
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:87
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:86
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:85
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:84
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:89
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:88
-; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:91
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:8
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:12
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:4
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:90
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:91
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:89
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:95
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:94
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:93
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:92
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:8
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:12
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:4
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:88
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:87
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:86
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:85
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:84
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:83
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:82
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:81
+; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:80
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:24
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:28
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:20
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:16
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:74
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:75
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:73
@@ -16674,10 +16669,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:66
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:65
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:64
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:92
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:104
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:108
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:96
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:58
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:59
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:57
@@ -16694,10 +16689,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:50
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:49
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:48
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:104
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:108
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:100
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:96
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:120
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:124
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:116
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:112
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:42
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:43
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:41
@@ -16714,10 +16709,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:34
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:33
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:32
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:56
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:60
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:52
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:48
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:72
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:76
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:68
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:64
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:26
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:27
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:25
@@ -16734,10 +16729,10 @@ define void @memset_p1_sz2048(ptr addrspace(1) %dst) #1 {
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:18
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:17
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:16
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:72
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:76
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:68
-; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:64
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:88
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:92
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:84
+; ALIGNED-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:80
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:10
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:11
; ALIGNED-NEXT: global_store_byte v[2:3], v5, off offset:9
@@ -17010,22 +17005,22 @@ define void @memset_p3_sz2048(ptr addrspace(3) %dst) #1 {
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:98
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:97
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:96
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:87
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:86
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:85
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:84
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:81
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:80
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:83
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:82
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:95
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:94
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:93
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:92
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:89
-; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:88
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:91
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:90
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:89
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:88
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:87
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:86
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:85
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:84
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:83
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:82
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:81
+; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:80
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:79
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:78
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:77
diff --git a/llvm/test/CodeGen/X86/buildvec-strided-loads.ll b/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
index db530a2618aa70..2ccf63ae5b86a7 100644
--- a/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
+++ b/llvm/test/CodeGen/X86/buildvec-strided-loads.ll
@@ -337,7 +337,7 @@ define <32 x i8> @buildvec_v32i8_stride4(ptr %p) {
; AVX2-NEXT: vpsrld $24, %ymm3, %ymm3
; AVX2-NEXT: vextracti128 $1, %ymm3, %xmm4
; AVX2-NEXT: vpackusdw %xmm4, %xmm3, %xmm3
-; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,u,u,u,u,u,u,u,u,16,17,20,21,24,25,28,29,u,u,u,u,u,u,u,u]
+; AVX2-NEXT: vmovdqa {{.*#+}} ymm4 = [0,1,4,5,8,9,12,13,8,9,12,13,12,13,14,15,16,17,20,21,24,25,28,29,24,25,28,29,28,29,30,31]
; AVX2-NEXT: vpshufb %ymm4, %ymm2, %ymm2
; AVX2-NEXT: vpermq {{.*#+}} ymm2 = ymm2[0,2,2,3]
; AVX2-NEXT: vinserti128 $1, %xmm3, %ymm2, %ymm2
diff --git a/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll b/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll
index 08b38bb8d4166e..b125ed94f3bc97 100644
--- a/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll
+++ b/llvm/test/CodeGen/X86/masked_gather_scatter_widen.ll
@@ -657,8 +657,7 @@ define <17 x float> @test_mgather_v17f32(ptr %base, <17 x i32> %index)
;
; WIDEN_AVX2-LABEL: test_mgather_v17f32:
; WIDEN_AVX2: # %bb.0:
-; WIDEN_AVX2-NEXT: vbroadcasti128 {{.*#+}} ymm0 = [0,2,4,6,0,2,4,6]
-; WIDEN_AVX2-NEXT: # ymm0 = mem[0,1,0,1]
+; WIDEN_AVX2-NEXT: vmovdqa {{.*#+}} ymm0 = [0,2,4,6,4,6,6,7]
; WIDEN_AVX2-NEXT: vpermd {{[0-9]+}}(%rsp), %ymm0, %ymm1
; WIDEN_AVX2-NEXT: movq %rdi, %rax
; WIDEN_AVX2-NEXT: vpermd {{[0-9]+}}(%rsp), %ymm0, %ymm2
diff --git a/llvm/test/CodeGen/X86/pr40730.ll b/llvm/test/CodeGen/X86/pr40730.ll
index 304d071e5d6e53..a878e863d69fc1 100644
--- a/llvm/test/CodeGen/X86/pr40730.ll
+++ b/llvm/test/CodeGen/X86/pr40730.ll
@@ -17,10 +17,14 @@ define <8 x i32> @shuffle_v8i32_0dcd3f14(<8 x i32> %a, <8 x i32> %b) {
}
; CHECK: .LCPI1_0:
-; CHECK-NEXT: .quad 0x0000000e0000000d
-; CHECK-NEXT: .quad 0x0000000e0000000d
-; CHECK-NEXT: .quad 0x0000001000000000
-; CHECK-NEXT: .zero 8
+; CHECK-NEXT: .zero 4
+; CHECK-NEXT: .zero 4
+; CHECK-NEXT: .zero 4
+; CHECK-NEXT: .zero 4
+; CHECK-NEXT: .long 13
+; CHECK-NEXT: .long 14
+; CHECK-NEXT: .zero 4
+; CHECK-NEXT: .long 16
define <8 x i32> @shuffle_v8i32_0dcd3f14_constant(<8 x i32> %a0) {
; CHECK-LABEL: shuffle_v8i32_0dcd3f14_constant:
@@ -29,7 +33,9 @@ define <8 x i32> @shuffle_v8i32_0dcd3f14_constant(<8 x i32> %a0) {
; CHECK-NEXT: vmovss {{.*#+}} xmm1 = xmm1[0],xmm0[1,2,3]
; CHECK-NEXT: vshufps {{.*#+}} xmm1 = xmm1[3,1,1,0]
; CHECK-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0
-; CHECK-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],mem[1,2,3],ymm0[4],mem[5],ymm0[6,7]
+; CHECK-NEXT: vbroadcastf128 {{.*#+}} ymm1 = mem[0,1,0,1]
+; CHECK-NEXT: vshufpd {{.*#+}} ymm1 = ymm1[0,0,3,2]
+; CHECK-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5],ymm0[6,7]
; CHECK-NEXT: retq
%res = shufflevector <8 x i32> %a0, <8 x i32> <i32 9, i32 10, i32 11, i32 12, i32 13, i32 14, i32 15, i32 16>, <8 x i32> <i32 0, i32 13, i32 12, i32 13, i32 3, i32 15, i32 1, i32 4>
ret <8 x i32> %res
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll
index 93d907786d791b..423cb1e1235da5 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-load-i16-stride-5.ll
@@ -8357,11 +8357,11 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512-FCP-LABEL: load_i16_stride5_vf64:
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: subq $648, %rsp # imm = 0x288
-; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm0 = [4,5,14,15,4,5,6,7,0,0,0,0,0,0,0,0]
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,14,15,4,5,6,7,8,9,10,11,12,13,14,15]
; AVX512-FCP-NEXT: vmovdqa64 496(%rdi), %xmm1
; AVX512-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm2
; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm31
-; AVX512-FCP-NEXT: vmovq {{.*#+}} xmm1 = [0,1,10,11,8,9,10,11,0,0,0,0,0,0,0,0]
+; AVX512-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
; AVX512-FCP-NEXT: vmovdqa64 480(%rdi), %xmm4
; AVX512-FCP-NEXT: vpshufb %xmm1, %xmm4, %xmm3
; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm21
@@ -9261,11 +9261,11 @@ define void @load_i16_stride5_vf64(ptr %in.vec, ptr %out.vec0, ptr %out.vec1, pt
; AVX512DQ-FCP-LABEL: load_i16_stride5_vf64:
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: subq $648, %rsp # imm = 0x288
-; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm0 = [4,5,14,15,4,5,6,7,0,0,0,0,0,0,0,0]
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm0 = [4,5,14,15,4,5,6,7,8,9,10,11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vmovdqa64 496(%rdi), %xmm1
; AVX512DQ-FCP-NEXT: vpshufb %xmm0, %xmm1, %xmm2
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm31
-; AVX512DQ-FCP-NEXT: vmovq {{.*#+}} xmm1 = [0,1,10,11,8,9,10,11,0,0,0,0,0,0,0,0]
+; AVX512DQ-FCP-NEXT: vmovdqa {{.*#+}} xmm1 = [0,1,10,11,8,9,10,11,8,9,10,11,12,13,14,15]
; AVX512DQ-FCP-NEXT: vmovdqa64 480(%rdi), %xmm4
; AVX512DQ-FCP-NEXT: vpshufb %xmm1, %xmm4, %xmm3
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm21
diff --git a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll
index cfd4b4808eb93e..df9bf001ce7431 100644
--- a/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll
+++ b/llvm/test/CodeGen/X86/vector-interleaved-store-i16-stride-8.ll
@@ -4219,124 +4219,125 @@ define void @store_i16_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP: # %bb.0:
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX512-FCP-NEXT: vmovdqa (%r10), %xmm1
-; AVX512-FCP-NEXT: vmovdqa (%rax), %xmm2
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm21
+; AVX512-FCP-NEXT: vmovdqa (%r10), %xmm0
+; AVX512-FCP-NEXT: vmovdqa (%rax), %xmm1
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm20
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm21
+; AVX512-FCP-NEXT: vmovdqa (%r9), %xmm0
+; AVX512-FCP-NEXT: vmovdqa (%r8), %xmm1
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm22
-; AVX512-FCP-NEXT: vmovdqa (%r9), %xmm1
-; AVX512-FCP-NEXT: vmovdqa (%r8), %xmm2
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm23
-; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm24
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm23
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]
-; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm16, %zmm5
-; AVX512-FCP-NEXT: vmovdqa (%rcx), %xmm1
-; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm2
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm25
+; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm16, %zmm5
+; AVX512-FCP-NEXT: vmovdqa (%rcx), %xmm0
+; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm1
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm24
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm25
+; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm0
+; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm1
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm9 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm26
-; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm1
-; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm9
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm27
-; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm19 = [0,16,1,0,1,17,0,0,2,18,3,0,3,19,0,0]
-; AVX512-FCP-NEXT: vpermi2d %zmm0, %zmm10, %zmm19
+; AVX512-FCP-NEXT: vmovdqa64 %xmm0, %xmm27
+; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
+; AVX512-FCP-NEXT: vpermt2d %zmm10, %zmm17, %zmm9
; AVX512-FCP-NEXT: movb $-86, %r11b
; AVX512-FCP-NEXT: kmovw %r11d, %k1
-; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm19 {%k1}
-; AVX512-FCP-NEXT: vmovdqa 32(%r10), %ymm1
-; AVX512-FCP-NEXT: vmovdqa 32(%rax), %ymm5
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm10 = ymm5[0],ymm1[0],ymm5[1],ymm1[1],ymm5[2],ymm1[2],ymm5[3],ymm1[3],ymm5[8],ymm1[8],ymm5[9],ymm1[9],ymm5[10],ymm1[10],ymm5[11],ymm1[11]
-; AVX512-FCP-NEXT: vmovdqa 32(%r9), %ymm12
-; AVX512-FCP-NEXT: vmovdqa 32(%r8), %ymm14
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm15 = ymm14[0],ymm12[0],ymm14[1],ymm12[1],ymm14[2],ymm12[2],ymm14[3],ymm12[3],ymm14[8],ymm12[8],ymm14[9],ymm12[9],ymm14[10],ymm12[10],ymm14[11],ymm12[11]
-; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
-; AVX512-FCP-NEXT: vpermt2d %zmm10, %zmm17, %zmm15
-; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %ymm11
-; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm13
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]
-; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
-; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm3
-; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm10 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[8],ymm3[8],ymm0[9],ymm3[9],ymm0[10],ymm3[10],ymm0[11],ymm3[11]
-; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm18, %zmm10
-; AVX512-FCP-NEXT: vmovdqa64 %zmm15, %zmm10 {%k1}
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm5[4],ymm1[4],ymm5[5],ymm1[5],ymm5[6],ymm1[6],ymm5[7],ymm1[7],ymm5[12],ymm1[12],ymm5[13],ymm1[13],ymm5[14],ymm1[14],ymm5[15],ymm1[15]
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm14[4],ymm12[4],ymm14[5],ymm12[5],ymm14[6],ymm12[6],ymm14[7],ymm12[7],ymm14[12],ymm12[12],ymm14[13],ymm12[13],ymm14[14],ymm12[14],ymm14[15],ymm12[15]
+; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm9 {%k1}
+; AVX512-FCP-NEXT: vmovdqa 32(%r10), %ymm5
+; AVX512-FCP-NEXT: vmovdqa 32(%rax), %ymm10
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm11 = ymm10[0],ymm5[0],ymm10[1],ymm5[1],ymm10[2],ymm5[2],ymm10[3],ymm5[3],ymm10[8],ymm5[8],ymm10[9],ymm5[9],ymm10[10],ymm5[10],ymm10[11],ymm5[11]
+; AVX512-FCP-NEXT: vmovdqa 32(%r9), %ymm13
+; AVX512-FCP-NEXT: vmovdqa 32(%r8), %ymm15
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm15[0],ymm13[0],ymm15[1],ymm13[1],ymm15[2],ymm13[2],ymm15[3],ymm13[3],ymm15[8],ymm13[8],ymm15[9],ymm13[9],ymm15[10],ymm13[10],ymm15[11],ymm13[11]
+; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
+; AVX512-FCP-NEXT: vpermt2d %zmm11, %zmm18, %zmm0
+; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %ymm3
+; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm12
+; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm3[0],ymm12[1],ymm3[1],ymm12[2],ymm3[2],ymm12[3],ymm3[3],ymm12[8],ymm3[8],ymm12[9],ymm3[9],ymm12[10],ymm3[10],ymm12[11],ymm3[11]
+; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %ymm2
+; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm11 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX512-FCP-NEXT: vpermt2d %zmm14, %zmm19, %zmm11
+; AVX512-FCP-NEXT: vmovdqa64 %zmm0, %zmm11 {%k1}
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm10[4],ymm5[4],ymm10[5],ymm5[5],ymm10[6],ymm5[6],ymm10[7],ymm5[7],ymm10[12],ymm5[12],ymm10[13],ymm5[13],ymm10[14],ymm5[14],ymm10[15],ymm5[15]
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm15[4],ymm13[4],ymm15[5],ymm13[5],ymm15[6],ymm13[6],ymm15[7],ymm13[7],ymm15[12],ymm13[12],ymm15[13],ymm13[13],ymm15[14],ymm13[14],ymm15[15],ymm13[15]
+; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm18, %zmm5
; AVX512-FCP-NEXT: vmovdqa 32(%r10), %xmm6
-; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm17, %zmm5
-; AVX512-FCP-NEXT: vmovdqa 32(%rax), %xmm4
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm11 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]
-; AVX512-FCP-NEXT: vmovdqa 32(%r9), %xmm13
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm12 = ymm0[4],ymm3[4],ymm0[5],ymm3[5],ymm0[6],ymm3[6],ymm0[7],ymm3[7],ymm0[12],ymm3[12],ymm0[13],ymm3[13],ymm0[14],ymm3[14],ymm0[15],ymm3[15]
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm12[4],ymm3[4],ymm12[5],ymm3[5],ymm12[6],ymm3[6],ymm12[7],ymm3[7],ymm12[12],ymm3[12],ymm12[13],ymm3[13],ymm12[14],ymm3[14],ymm12[15],ymm3[15]
+; AVX512-FCP-NEXT: vmovdqa 32(%rax), %xmm10
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm13 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX512-FCP-NEXT: vmovdqa 32(%r9), %xmm4
+; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm19, %zmm13
; AVX512-FCP-NEXT: vmovdqa 32(%r8), %xmm2
-; AVX512-FCP-NEXT: vpermt2d %zmm11, %zmm18, %zmm12
-; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm12 {%k1}
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm13[0],xmm2[1],xmm13[1],xmm2[2],xmm13[2],xmm2[3],xmm13[3]
+; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm13 {%k1}
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm16, %zmm5
; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %xmm3
-; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm20 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
-; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %xmm11
-; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %xmm15
+; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %xmm12
+; AVX512-FCP-NEXT: vmovdqa 32(%rsi), %xmm14
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm0
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm11[0],xmm3[0],xmm11[1],xmm3[1],xmm11[2],xmm3[2],xmm11[3],xmm3[3]
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
-; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm20, %zmm14
-; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm14 {%k1}
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm13[4],xmm2[5],xmm13[5],xmm2[6],xmm13[6],xmm2[7],xmm13[7]
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3]
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm15
+; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm15 {%k1}
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm16, %zmm2
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm3[4],xmm11[5],xmm3[5],xmm11[6],xmm3[6],xmm11[7],xmm3[7]
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]
-; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm20, %zmm5
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
+; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm5
; AVX512-FCP-NEXT: vmovdqa64 %zmm2, %zmm5 {%k1}
; AVX512-FCP-NEXT: vmovdqa (%r10), %ymm8
; AVX512-FCP-NEXT: vmovdqa (%rax), %ymm7
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]
; AVX512-FCP-NEXT: vmovdqa (%r9), %ymm3
; AVX512-FCP-NEXT: vmovdqa (%r8), %ymm4
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm17, %zmm6
-; AVX512-FCP-NEXT: vmovdqa (%rcx), %ymm11
-; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm13
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm15 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]
-; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm2
-; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
-; AVX512-FCP-NEXT: vpermt2d %zmm15, %zmm18, %zmm0
-; AVX512-FCP-NEXT: vmovdqa64 %zmm6, %zmm0 {%k1}
+; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm18, %zmm6
+; AVX512-FCP-NEXT: vmovdqa (%rcx), %ymm2
+; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm12
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm2[0],ymm12[1],ymm2[1],ymm12[2],ymm2[2],ymm12[3],ymm2[3],ymm12[8],ymm2[8],ymm12[9],ymm2[9],ymm12[10],ymm2[10],ymm12[11],ymm2[11]
+; AVX512-FCP-NEXT: vmovdqa (%rsi), %ymm1
+; AVX512-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm10 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; AVX512-FCP-NEXT: vpermt2d %zmm14, %zmm19, %zmm10
+; AVX512-FCP-NEXT: vmovdqa64 %zmm6, %zmm10 {%k1}
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm6 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512-FCP-NEXT: vpermt2d %zmm6, %zmm17, %zmm3
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
-; AVX512-FCP-NEXT: vpermt2d %zmm4, %zmm18, %zmm1
-; AVX512-FCP-NEXT: vmovdqa64 %zmm3, %zmm1 {%k1}
+; AVX512-FCP-NEXT: vpermt2d %zmm6, %zmm18, %zmm3
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm12[4],ymm2[4],ymm12[5],ymm2[5],ymm12[6],ymm2[6],ymm12[7],ymm2[7],ymm12[12],ymm2[12],ymm12[13],ymm2[13],ymm12[14],ymm2[14],ymm12[15],ymm2[15]
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
+; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm19, %zmm0
+; AVX512-FCP-NEXT: vmovdqa64 %zmm3, %zmm0 {%k1}
+; AVX512-FCP-NEXT: vmovdqa64 %xmm20, %xmm1
; AVX512-FCP-NEXT: vmovdqa64 %xmm21, %xmm2
-; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm3
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm2
; AVX512-FCP-NEXT: vmovdqa64 %xmm23, %xmm3
-; AVX512-FCP-NEXT: vmovdqa64 %xmm24, %xmm4
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm16, %zmm3
-; AVX512-FCP-NEXT: vmovdqa64 %xmm25, %xmm2
-; AVX512-FCP-NEXT: vmovdqa64 %xmm26, %xmm4
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm16, %zmm2
+; AVX512-FCP-NEXT: vmovdqa64 %xmm24, %xmm1
+; AVX512-FCP-NEXT: vmovdqa64 %xmm25, %xmm3
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512-FCP-NEXT: vmovdqa64 %xmm26, %xmm3
; AVX512-FCP-NEXT: vmovdqa64 %xmm27, %xmm4
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm20, %zmm4
-; AVX512-FCP-NEXT: vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm3
+; AVX512-FCP-NEXT: vmovdqa64 %zmm2, %zmm3 {%k1}
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512-FCP-NEXT: vmovdqa64 %zmm4, (%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm1, 192(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm0, 128(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm3, (%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm0, 192(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm10, 128(%rax)
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, 320(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm14, 256(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm12, 448(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm10, 384(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm19, 64(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm15, 256(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm13, 448(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm11, 384(%rax)
+; AVX512-FCP-NEXT: vmovdqa64 %zmm9, 64(%rax)
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
@@ -4470,124 +4471,125 @@ define void @store_i16_stride8_vf32(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP: # %bb.0:
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
-; AVX512DQ-FCP-NEXT: vmovdqa (%r10), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa (%rax), %xmm2
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm21
+; AVX512DQ-FCP-NEXT: vmovdqa (%r10), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa (%rax), %xmm1
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm20
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm21
+; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %xmm1
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm22
-; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %xmm2
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm23
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm24
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm23
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm16, %zmm5
-; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm2
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm25
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm16, %zmm5
+; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm1
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm24
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm25
+; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm0
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm1
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm9 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm26
-; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm1
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm9
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm10 = xmm9[4],xmm1[4],xmm9[5],xmm1[5],xmm9[6],xmm1[6],xmm9[7],xmm1[7]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm27
-; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm19 = [0,16,1,0,1,17,0,0,2,18,3,0,3,19,0,0]
-; AVX512DQ-FCP-NEXT: vpermi2d %zmm0, %zmm10, %zmm19
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm0, %xmm27
+; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm10, %zmm17, %zmm9
; AVX512DQ-FCP-NEXT: movb $-86, %r11b
; AVX512DQ-FCP-NEXT: kmovw %r11d, %k1
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm19 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%r10), %ymm1
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %ymm5
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm10 = ymm5[0],ymm1[0],ymm5[1],ymm1[1],ymm5[2],ymm1[2],ymm5[3],ymm1[3],ymm5[8],ymm1[8],ymm5[9],ymm1[9],ymm5[10],ymm1[10],ymm5[11],ymm1[11]
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %ymm12
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %ymm14
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm15 = ymm14[0],ymm12[0],ymm14[1],ymm12[1],ymm14[2],ymm12[2],ymm14[3],ymm12[3],ymm14[8],ymm12[8],ymm14[9],ymm12[9],ymm14[10],ymm12[10],ymm14[11],ymm12[11]
-; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm10, %zmm17, %zmm15
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %ymm11
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm13
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]
-; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm3
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm0
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm10 = ymm0[0],ymm3[0],ymm0[1],ymm3[1],ymm0[2],ymm3[2],ymm0[3],ymm3[3],ymm0[8],ymm3[8],ymm0[9],ymm3[9],ymm0[10],ymm3[10],ymm0[11],ymm3[11]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm18, %zmm10
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm15, %zmm10 {%k1}
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm5[4],ymm1[4],ymm5[5],ymm1[5],ymm5[6],ymm1[6],ymm5[7],ymm1[7],ymm5[12],ymm1[12],ymm5[13],ymm1[13],ymm5[14],ymm1[14],ymm5[15],ymm1[15]
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm14[4],ymm12[4],ymm14[5],ymm12[5],ymm14[6],ymm12[6],ymm14[7],ymm12[7],ymm14[12],ymm12[12],ymm14[13],ymm12[13],ymm14[14],ymm12[14],ymm14[15],ymm12[15]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm9 {%k1}
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%r10), %ymm5
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %ymm10
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm11 = ymm10[0],ymm5[0],ymm10[1],ymm5[1],ymm10[2],ymm5[2],ymm10[3],ymm5[3],ymm10[8],ymm5[8],ymm10[9],ymm5[9],ymm10[10],ymm5[10],ymm10[11],ymm5[11]
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %ymm13
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %ymm15
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm15[0],ymm13[0],ymm15[1],ymm13[1],ymm15[2],ymm13[2],ymm15[3],ymm13[3],ymm15[8],ymm13[8],ymm15[9],ymm13[9],ymm15[10],ymm13[10],ymm15[11],ymm13[11]
+; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm11, %zmm18, %zmm0
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %ymm3
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm12
+; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm3[0],ymm12[1],ymm3[1],ymm12[2],ymm3[2],ymm12[3],ymm3[3],ymm12[8],ymm3[8],ymm12[9],ymm3[9],ymm12[10],ymm3[10],ymm12[11],ymm3[11]
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %ymm1
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm11 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm14, %zmm19, %zmm11
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, %zmm11 {%k1}
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm10[4],ymm5[4],ymm10[5],ymm5[5],ymm10[6],ymm5[6],ymm10[7],ymm5[7],ymm10[12],ymm5[12],ymm10[13],ymm5[13],ymm10[14],ymm5[14],ymm10[15],ymm5[15]
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm15[4],ymm13[4],ymm15[5],ymm13[5],ymm15[6],ymm13[6],ymm15[7],ymm13[7],ymm15[12],ymm13[12],ymm15[13],ymm13[13],ymm15[14],ymm13[14],ymm15[15],ymm13[15]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm18, %zmm5
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r10), %xmm6
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm17, %zmm5
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %xmm4
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm11 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %xmm13
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm12 = ymm0[4],ymm3[4],ymm0[5],ymm3[5],ymm0[6],ymm3[6],ymm0[7],ymm3[7],ymm0[12],ymm3[12],ymm0[13],ymm3[13],ymm0[14],ymm3[14],ymm0[15],ymm3[15]
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm12[4],ymm3[4],ymm12[5],ymm3[5],ymm12[6],ymm3[6],ymm12[7],ymm3[7],ymm12[12],ymm3[12],ymm12[13],ymm3[13],ymm12[14],ymm3[14],ymm12[15],ymm3[15]
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %xmm10
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm13 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %xmm4
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm19, %zmm13
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %xmm2
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm11, %zmm18, %zmm12
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm12 {%k1}
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm4[0],xmm6[0],xmm4[1],xmm6[1],xmm4[2],xmm6[2],xmm4[3],xmm6[3]
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm13[0],xmm2[1],xmm13[1],xmm2[2],xmm13[2],xmm2[3],xmm13[3]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm13 {%k1}
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm10[0],xmm6[0],xmm10[1],xmm6[1],xmm10[2],xmm6[2],xmm10[3],xmm6[3]
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm16, %zmm5
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %xmm3
-; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm20 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %xmm11
-; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %xmm15
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %xmm12
+; AVX512DQ-FCP-NEXT: vmovdqa 32(%rsi), %xmm14
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm0
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm11[0],xmm3[0],xmm11[1],xmm3[1],xmm11[2],xmm3[2],xmm11[3],xmm3[3]
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm14 = xmm0[0],xmm15[0],xmm0[1],xmm15[1],xmm0[2],xmm15[2],xmm0[3],xmm15[3]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm20, %zmm14
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm14 {%k1}
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm6[4],xmm4[5],xmm6[5],xmm4[6],xmm6[6],xmm4[7],xmm6[7]
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm13[4],xmm2[5],xmm13[5],xmm2[6],xmm13[6],xmm2[7],xmm13[7]
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm12[0],xmm3[0],xmm12[1],xmm3[1],xmm12[2],xmm3[2],xmm12[3],xmm3[3]
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm15 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm15
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm15 {%k1}
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm10[4],xmm6[4],xmm10[5],xmm6[5],xmm10[6],xmm6[6],xmm10[7],xmm6[7]
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm2[4],xmm4[4],xmm2[5],xmm4[5],xmm2[6],xmm4[6],xmm2[7],xmm4[7]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm16, %zmm2
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm11[4],xmm3[4],xmm11[5],xmm3[5],xmm11[6],xmm3[6],xmm11[7],xmm3[7]
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm15[4],xmm0[5],xmm15[5],xmm0[6],xmm15[6],xmm0[7],xmm15[7]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm20, %zmm5
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm12[4],xmm3[4],xmm12[5],xmm3[5],xmm12[6],xmm3[6],xmm12[7],xmm3[7]
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm5 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm5
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, %zmm5 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa (%r10), %ymm8
; AVX512DQ-FCP-NEXT: vmovdqa (%rax), %ymm7
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm7[0],ymm8[0],ymm7[1],ymm8[1],ymm7[2],ymm8[2],ymm7[3],ymm8[3],ymm7[8],ymm8[8],ymm7[9],ymm8[9],ymm7[10],ymm8[10],ymm7[11],ymm8[11]
; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %ymm3
; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %ymm4
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm17, %zmm6
-; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %ymm11
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm13
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm15 = ymm13[0],ymm11[0],ymm13[1],ymm11[1],ymm13[2],ymm11[2],ymm13[3],ymm11[3],ymm13[8],ymm11[8],ymm13[9],ymm11[9],ymm13[10],ymm11[10],ymm13[11],ymm11[11]
-; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm2
-; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm1
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm0 = ymm1[0],ymm2[0],ymm1[1],ymm2[1],ymm1[2],ymm2[2],ymm1[3],ymm2[3],ymm1[8],ymm2[8],ymm1[9],ymm2[9],ymm1[10],ymm2[10],ymm1[11],ymm2[11]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm15, %zmm18, %zmm0
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, %zmm0 {%k1}
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm18, %zmm6
+; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %ymm2
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm12
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm14 = ymm12[0],ymm2[0],ymm12[1],ymm2[1],ymm12[2],ymm2[2],ymm12[3],ymm2[3],ymm12[8],ymm2[8],ymm12[9],ymm2[9],ymm12[10],ymm2[10],ymm12[11],ymm2[11]
+; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %ymm1
+; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm10 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm14, %zmm19, %zmm10
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, %zmm10 {%k1}
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm6 = ymm7[4],ymm8[4],ymm7[5],ymm8[5],ymm7[6],ymm8[6],ymm7[7],ymm8[7],ymm7[12],ymm8[12],ymm7[13],ymm8[13],ymm7[14],ymm8[14],ymm7[15],ymm8[15]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm6, %zmm17, %zmm3
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm13[4],ymm11[4],ymm13[5],ymm11[5],ymm13[6],ymm11[6],ymm13[7],ymm11[7],ymm13[12],ymm11[12],ymm13[13],ymm11[13],ymm13[14],ymm11[14],ymm13[15],ymm11[15]
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm4, %zmm18, %zmm1
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, %zmm1 {%k1}
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm6, %zmm18, %zmm3
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm12[4],ymm2[4],ymm12[5],ymm2[5],ymm12[6],ymm2[6],ymm12[7],ymm2[7],ymm12[12],ymm2[12],ymm12[13],ymm2[13],ymm12[14],ymm2[14],ymm12[15],ymm2[15]
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm19, %zmm0
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, %zmm0 {%k1}
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm20, %xmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm21, %xmm2
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm3
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm2[0],xmm1[1],xmm2[1],xmm1[2],xmm2[2],xmm1[3],xmm2[3]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm2
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm23, %xmm3
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm24, %xmm4
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm16, %zmm3
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm25, %xmm2
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm26, %xmm4
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm4[0],xmm2[1],xmm4[1],xmm2[2],xmm4[2],xmm2[3],xmm4[3]
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm16, %zmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm24, %xmm1
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm25, %xmm3
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1],xmm1[2],xmm3[2],xmm1[3],xmm3[3]
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm26, %xmm3
; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm27, %xmm4
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm9[0],xmm4[0],xmm9[1],xmm4[1],xmm9[2],xmm4[2],xmm9[3],xmm4[3]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm20, %zmm4
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, %zmm4 {%k1}
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm4[0],xmm3[1],xmm4[1],xmm3[2],xmm4[2],xmm3[3],xmm4[3]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, %zmm3 {%k1}
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm4, (%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, 192(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, 128(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, (%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm0, 192(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm10, 128(%rax)
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, 320(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm14, 256(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm12, 448(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm10, 384(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm19, 64(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm15, 256(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm13, 448(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm11, 384(%rax)
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm9, 64(%rax)
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
@@ -7957,57 +7959,58 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
;
; AVX512-FCP-LABEL: store_i16_stride8_vf64:
; AVX512-FCP: # %bb.0:
-; AVX512-FCP-NEXT: subq $328, %rsp # imm = 0x148
+; AVX512-FCP-NEXT: subq $392, %rsp # imm = 0x188
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512-FCP-NEXT: vmovdqa (%r10), %xmm1
; AVX512-FCP-NEXT: vmovdqa (%rax), %xmm2
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm21
-; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm22
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm20
+; AVX512-FCP-NEXT: vmovdqa64 %xmm1, %xmm21
; AVX512-FCP-NEXT: vmovdqa (%r9), %xmm2
; AVX512-FCP-NEXT: vmovdqa (%r8), %xmm3
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm3, %xmm23
-; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm24
+; AVX512-FCP-NEXT: vmovdqa64 %xmm3, %xmm22
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm23
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]
; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm16, %zmm1
; AVX512-FCP-NEXT: vmovdqa (%rcx), %xmm2
; AVX512-FCP-NEXT: vmovdqa (%rdx), %xmm3
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm3, %xmm25
-; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm26
+; AVX512-FCP-NEXT: vmovdqa64 %xmm3, %xmm24
+; AVX512-FCP-NEXT: vmovdqa64 %xmm2, %xmm25
; AVX512-FCP-NEXT: vmovdqa (%rsi), %xmm3
; AVX512-FCP-NEXT: vmovdqa (%rdi), %xmm4
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm27
-; AVX512-FCP-NEXT: vmovdqa64 %xmm3, %xmm28
-; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm20 = [0,16,1,0,1,17,0,0,2,18,3,0,3,19,0,0]
-; AVX512-FCP-NEXT: vpermi2d %zmm0, %zmm2, %zmm20
+; AVX512-FCP-NEXT: vmovdqa64 %xmm4, %xmm26
+; AVX512-FCP-NEXT: vmovdqa64 %xmm3, %xmm27
+; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
+; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm17, %zmm2
; AVX512-FCP-NEXT: movb $-86, %r11b
; AVX512-FCP-NEXT: kmovw %r11d, %k1
-; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm20 {%k1}
+; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 96(%r10), %ymm0
; AVX512-FCP-NEXT: vmovdqa 96(%rax), %ymm1
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
; AVX512-FCP-NEXT: vmovdqa 96(%r9), %ymm3
; AVX512-FCP-NEXT: vmovdqa 96(%r8), %ymm4
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
-; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm17, %zmm5
+; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
+; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm18, %zmm5
; AVX512-FCP-NEXT: vmovdqa 96(%rcx), %ymm2
; AVX512-FCP-NEXT: vmovdqa 96(%rdx), %ymm6
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm11 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]
; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm10 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]
; AVX512-FCP-NEXT: vmovdqa 96(%rsi), %ymm12
; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %ymm13
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]
-; AVX512-FCP-NEXT: vpermt2d %zmm11, %zmm19, %zmm7
+; AVX512-FCP-NEXT: vpermt2d %zmm10, %zmm19, %zmm7
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm7 {%k1}
; AVX512-FCP-NEXT: vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm17, %zmm1
+; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm18, %zmm1
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm6[4],ymm2[4],ymm6[5],ymm2[5],ymm6[6],ymm2[6],ymm6[7],ymm2[7],ymm6[12],ymm2[12],ymm6[13],ymm2[13],ymm6[14],ymm2[14],ymm6[15],ymm2[15]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]
; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm19, %zmm2
@@ -8021,21 +8024,20 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm16, %zmm5
; AVX512-FCP-NEXT: vmovdqa 96(%rcx), %xmm2
-; AVX512-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
; AVX512-FCP-NEXT: vmovdqa 96(%rdx), %xmm6
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
; AVX512-FCP-NEXT: vmovdqa 96(%rsi), %xmm13
-; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm15
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]
-; AVX512-FCP-NEXT: vpermt2d %zmm12, %zmm18, %zmm7
+; AVX512-FCP-NEXT: vmovdqa 96(%rdi), %xmm14
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512-FCP-NEXT: vpermt2d %zmm10, %zmm17, %zmm7
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm7 {%k1}
; AVX512-FCP-NEXT: vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm16, %zmm1
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm15[4],xmm13[4],xmm15[5],xmm13[5],xmm15[6],xmm13[6],xmm15[7],xmm13[7]
-; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm18, %zmm2
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
+; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm17, %zmm2
; AVX512-FCP-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
; AVX512-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512-FCP-NEXT: vmovdqa 64(%r10), %ymm6
@@ -8043,47 +8045,47 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]
; AVX512-FCP-NEXT: vmovdqa 64(%r9), %ymm3
; AVX512-FCP-NEXT: vmovdqa 64(%r8), %ymm4
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm12 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm17, %zmm12
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm10 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm18, %zmm10
; AVX512-FCP-NEXT: vmovdqa 64(%rcx), %ymm2
; AVX512-FCP-NEXT: vmovdqa 64(%rdx), %ymm13
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm15 = ymm13[0],ymm2[0],ymm13[1],ymm2[1],ymm13[2],ymm2[2],ymm13[3],ymm2[3],ymm13[8],ymm2[8],ymm13[9],ymm2[9],ymm13[10],ymm2[10],ymm13[11],ymm2[11]
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm14 = ymm13[0],ymm2[0],ymm13[1],ymm2[1],ymm13[2],ymm2[2],ymm13[3],ymm2[3],ymm13[8],ymm2[8],ymm13[9],ymm2[9],ymm13[10],ymm2[10],ymm13[11],ymm2[11]
; AVX512-FCP-NEXT: vmovdqa 64(%rsi), %ymm1
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %ymm0
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; AVX512-FCP-NEXT: vpermt2d %zmm15, %zmm19, %zmm7
-; AVX512-FCP-NEXT: vmovdqa64 %zmm12, %zmm7 {%k1}
+; AVX512-FCP-NEXT: vpermt2d %zmm14, %zmm19, %zmm7
+; AVX512-FCP-NEXT: vmovdqa64 %zmm10, %zmm7 {%k1}
; AVX512-FCP-NEXT: vmovdqu64 %zmm7, (%rsp) # 64-byte Spill
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512-FCP-NEXT: vpermt2d %zmm5, %zmm17, %zmm3
+; AVX512-FCP-NEXT: vpermt2d %zmm5, %zmm18, %zmm3
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm2[4],ymm13[5],ymm2[5],ymm13[6],ymm2[6],ymm13[7],ymm2[7],ymm13[12],ymm2[12],ymm13[13],ymm2[13],ymm13[14],ymm2[14],ymm13[15],ymm2[15]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm19, %zmm0
-; AVX512-FCP-NEXT: vmovdqa 64(%r10), %xmm8
; AVX512-FCP-NEXT: vmovdqa64 %zmm3, %zmm0 {%k1}
; AVX512-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512-FCP-NEXT: vmovdqa 64(%r10), %xmm7
; AVX512-FCP-NEXT: vmovdqa 64(%rax), %xmm2
-; AVX512-FCP-NEXT: vmovdqa 64(%r9), %xmm7
+; AVX512-FCP-NEXT: vmovdqa 64(%r9), %xmm8
; AVX512-FCP-NEXT: vmovdqa 64(%r8), %xmm3
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3]
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3]
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3]
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]
; AVX512-FCP-NEXT: vpermt2d %zmm4, %zmm16, %zmm5
; AVX512-FCP-NEXT: vmovdqa 64(%rcx), %xmm4
; AVX512-FCP-NEXT: vmovdqa 64(%rdx), %xmm6
-; AVX512-FCP-NEXT: vmovdqa 64(%rsi), %xmm13
+; AVX512-FCP-NEXT: vmovdqa 64(%rsi), %xmm14
; AVX512-FCP-NEXT: vmovdqa 64(%rdi), %xmm0
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
-; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm18, %zmm9
+; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm9
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm9 {%k1}
; AVX512-FCP-NEXT: vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7]
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]
; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm16, %zmm2
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
-; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]
-; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm18, %zmm15
+; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
+; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm15
; AVX512-FCP-NEXT: vmovdqa64 %zmm2, %zmm15 {%k1}
; AVX512-FCP-NEXT: vmovdqa 32(%r10), %ymm8
; AVX512-FCP-NEXT: vmovdqa 32(%rax), %ymm4
@@ -8091,7 +8093,7 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa 32(%r9), %ymm3
; AVX512-FCP-NEXT: vmovdqa 32(%r8), %ymm5
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]
-; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm17, %zmm6
+; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm18, %zmm6
; AVX512-FCP-NEXT: vmovdqa 32(%rcx), %ymm2
; AVX512-FCP-NEXT: vmovdqa 32(%rdx), %ymm7
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm9 = ymm7[0],ymm2[0],ymm7[1],ymm2[1],ymm7[2],ymm2[2],ymm7[3],ymm2[3],ymm7[8],ymm2[8],ymm7[9],ymm2[9],ymm7[10],ymm2[10],ymm7[11],ymm2[11]
@@ -8102,7 +8104,7 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 %zmm6, %zmm14 {%k1}
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm8[4],ymm4[5],ymm8[5],ymm4[6],ymm8[6],ymm4[7],ymm8[7],ymm4[12],ymm8[12],ymm4[13],ymm8[13],ymm4[14],ymm8[14],ymm4[15],ymm8[15]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm3[4],ymm5[5],ymm3[5],ymm5[6],ymm3[6],ymm5[7],ymm3[7],ymm5[12],ymm3[12],ymm5[13],ymm3[13],ymm5[14],ymm3[14],ymm5[15],ymm3[15]
-; AVX512-FCP-NEXT: vpermt2d %zmm4, %zmm17, %zmm5
+; AVX512-FCP-NEXT: vpermt2d %zmm4, %zmm18, %zmm5
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm7[4],ymm2[4],ymm7[5],ymm2[5],ymm7[6],ymm2[6],ymm7[7],ymm2[7],ymm7[12],ymm2[12],ymm7[13],ymm2[13],ymm7[14],ymm2[14],ymm7[15],ymm2[15]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm13 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm19, %zmm13
@@ -8120,14 +8122,14 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa 32(%rdi), %xmm0
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]
-; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm18, %zmm12
+; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm12
; AVX512-FCP-NEXT: vmovdqa64 %zmm6, %zmm12 {%k1}
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm16, %zmm3
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]
-; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm18, %zmm4
+; AVX512-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm4
; AVX512-FCP-NEXT: vmovdqa64 %zmm3, %zmm4 {%k1}
; AVX512-FCP-NEXT: vmovdqa (%r10), %ymm10
; AVX512-FCP-NEXT: vmovdqa (%rax), %ymm3
@@ -8135,7 +8137,7 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa (%r9), %ymm5
; AVX512-FCP-NEXT: vmovdqa (%r8), %ymm6
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]
-; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm17, %zmm7
+; AVX512-FCP-NEXT: vpermt2d %zmm0, %zmm18, %zmm7
; AVX512-FCP-NEXT: vmovdqa (%rcx), %ymm8
; AVX512-FCP-NEXT: vmovdqa (%rdx), %ymm9
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} ymm11 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]
@@ -8146,25 +8148,25 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovdqa64 %zmm7, %zmm0 {%k1}
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm10[4],ymm3[5],ymm10[5],ymm3[6],ymm10[6],ymm3[7],ymm10[7],ymm3[12],ymm10[12],ymm3[13],ymm10[13],ymm3[14],ymm10[14],ymm3[15],ymm10[15]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]
-; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm17, %zmm5
+; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm18, %zmm5
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]
; AVX512-FCP-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
; AVX512-FCP-NEXT: vpermt2d %zmm3, %zmm19, %zmm1
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, %zmm1 {%k1}
-; AVX512-FCP-NEXT: vmovdqa64 %xmm21, %xmm2
-; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm3
+; AVX512-FCP-NEXT: vmovdqa64 %xmm20, %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 %xmm21, %xmm3
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm23, %xmm3
-; AVX512-FCP-NEXT: vmovdqa64 %xmm24, %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 %xmm22, %xmm3
+; AVX512-FCP-NEXT: vmovdqa64 %xmm23, %xmm5
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm16, %zmm3
-; AVX512-FCP-NEXT: vmovdqa64 %xmm25, %xmm2
-; AVX512-FCP-NEXT: vmovdqa64 %xmm26, %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 %xmm24, %xmm2
+; AVX512-FCP-NEXT: vmovdqa64 %xmm25, %xmm5
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
-; AVX512-FCP-NEXT: vmovdqa64 %xmm27, %xmm5
-; AVX512-FCP-NEXT: vmovdqa64 %xmm28, %xmm6
+; AVX512-FCP-NEXT: vmovdqa64 %xmm26, %xmm5
+; AVX512-FCP-NEXT: vmovdqa64 %xmm27, %xmm6
; AVX512-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
-; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm18, %zmm5
+; AVX512-FCP-NEXT: vpermt2d %zmm2, %zmm17, %zmm5
; AVX512-FCP-NEXT: vmovdqa64 %zmm3, %zmm5 {%k1}
; AVX512-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512-FCP-NEXT: vmovdqa64 %zmm5, (%rax)
@@ -8189,8 +8191,9 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512-FCP-NEXT: vmovaps %zmm0, 960(%rax)
; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512-FCP-NEXT: vmovaps %zmm0, 896(%rax)
-; AVX512-FCP-NEXT: vmovdqa64 %zmm20, 64(%rax)
-; AVX512-FCP-NEXT: addq $328, %rsp # imm = 0x148
+; AVX512-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512-FCP-NEXT: vmovaps %zmm0, 64(%rax)
+; AVX512-FCP-NEXT: addq $392, %rsp # imm = 0x188
; AVX512-FCP-NEXT: vzeroupper
; AVX512-FCP-NEXT: retq
;
@@ -8436,57 +8439,58 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
;
; AVX512DQ-FCP-LABEL: store_i16_stride8_vf64:
; AVX512DQ-FCP: # %bb.0:
-; AVX512DQ-FCP-NEXT: subq $328, %rsp # imm = 0x148
+; AVX512DQ-FCP-NEXT: subq $392, %rsp # imm = 0x188
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %r10
; AVX512DQ-FCP-NEXT: vmovdqa (%r10), %xmm1
; AVX512DQ-FCP-NEXT: vmovdqa (%rax), %xmm2
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm21
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm22
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm20
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm1, %xmm21
; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %xmm2
; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %xmm3
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, %xmm23
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm24
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, %xmm22
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm23
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm16 = [0,0,0,16,0,0,1,17,2,2,2,18,0,0,3,19]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm16, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %xmm2
; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %xmm3
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm3[4],xmm2[4],xmm3[5],xmm2[5],xmm3[6],xmm2[6],xmm3[7],xmm2[7]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, %xmm25
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm26
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, %xmm24
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm2, %xmm25
; AVX512DQ-FCP-NEXT: vmovdqa (%rsi), %xmm3
; AVX512DQ-FCP-NEXT: vmovdqa (%rdi), %xmm4
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm27
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, %xmm28
-; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm20 = [0,16,1,0,1,17,0,0,2,18,3,0,3,19,0,0]
-; AVX512DQ-FCP-NEXT: vpermi2d %zmm0, %zmm2, %zmm20
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm4, %xmm26
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm3, %xmm27
+; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm17, %zmm2
; AVX512DQ-FCP-NEXT: movb $-86, %r11b
; AVX512DQ-FCP-NEXT: kmovw %r11d, %k1
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm20 {%k1}
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
+; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 96(%r10), %ymm0
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rax), %ymm1
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm1[0],ymm0[0],ymm1[1],ymm0[1],ymm1[2],ymm0[2],ymm1[3],ymm0[3],ymm1[8],ymm0[8],ymm1[9],ymm0[9],ymm1[10],ymm0[10],ymm1[11],ymm0[11]
; AVX512DQ-FCP-NEXT: vmovdqa 96(%r9), %ymm3
; AVX512DQ-FCP-NEXT: vmovdqa 96(%r8), %ymm4
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm5 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm17 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm17, %zmm5
+; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [0,1,4,20,4,5,5,21,2,1,6,22,6,5,7,23]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm18, %zmm5
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rcx), %ymm2
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdx), %ymm6
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm11 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]
; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm19 = [4,20,1,3,5,21,5,7,6,22,3,3,7,23,7,7]
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm10 = ymm6[0],ymm2[0],ymm6[1],ymm2[1],ymm6[2],ymm2[2],ymm6[3],ymm2[3],ymm6[8],ymm2[8],ymm6[9],ymm2[9],ymm6[10],ymm2[10],ymm6[11],ymm2[11]
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rsi), %ymm12
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %ymm13
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm13[0],ymm12[0],ymm13[1],ymm12[1],ymm13[2],ymm12[2],ymm13[3],ymm12[3],ymm13[8],ymm12[8],ymm13[9],ymm12[9],ymm13[10],ymm12[10],ymm13[11],ymm12[11]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm11, %zmm19, %zmm7
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm10, %zmm19, %zmm7
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm7 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm1[4],ymm0[4],ymm1[5],ymm0[5],ymm1[6],ymm0[6],ymm1[7],ymm0[7],ymm1[12],ymm0[12],ymm1[13],ymm0[13],ymm1[14],ymm0[14],ymm1[15],ymm0[15]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm17, %zmm1
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm18, %zmm1
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm6[4],ymm2[4],ymm6[5],ymm2[5],ymm6[6],ymm2[6],ymm6[7],ymm2[7],ymm6[12],ymm2[12],ymm6[13],ymm2[13],ymm6[14],ymm2[14],ymm6[15],ymm2[15]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm12[4],ymm13[5],ymm12[5],ymm13[6],ymm12[6],ymm13[7],ymm12[7],ymm13[12],ymm12[12],ymm13[13],ymm12[13],ymm13[14],ymm12[14],ymm13[15],ymm12[15]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm19, %zmm2
@@ -8500,21 +8504,20 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm4[0],xmm3[0],xmm4[1],xmm3[1],xmm4[2],xmm3[2],xmm4[3],xmm3[3]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm16, %zmm5
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rcx), %xmm2
-; AVX512DQ-FCP-NEXT: vpmovsxbd {{.*#+}} zmm18 = [0,16,1,0,1,17,0,0,2,18,3,3,3,19,0,0]
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdx), %xmm6
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm10 = xmm6[0],xmm2[0],xmm6[1],xmm2[1],xmm6[2],xmm2[2],xmm6[3],xmm2[3]
; AVX512DQ-FCP-NEXT: vmovdqa 96(%rsi), %xmm13
-; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm15
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm15[0],xmm13[0],xmm15[1],xmm13[1],xmm15[2],xmm13[2],xmm15[3],xmm13[3]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm12, %zmm18, %zmm7
+; AVX512DQ-FCP-NEXT: vmovdqa 96(%rdi), %xmm14
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm7 = xmm14[0],xmm13[0],xmm14[1],xmm13[1],xmm14[2],xmm13[2],xmm14[3],xmm13[3]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm10, %zmm17, %zmm7
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm7 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm7, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm1[4],xmm0[4],xmm1[5],xmm0[5],xmm1[6],xmm0[6],xmm1[7],xmm0[7]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm4[4],xmm3[4],xmm4[5],xmm3[5],xmm4[6],xmm3[6],xmm4[7],xmm3[7]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm16, %zmm1
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm6[4],xmm2[4],xmm6[5],xmm2[5],xmm6[6],xmm2[6],xmm6[7],xmm2[7]
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm15[4],xmm13[4],xmm15[5],xmm13[5],xmm15[6],xmm13[6],xmm15[7],xmm13[7]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm18, %zmm2
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm14[4],xmm13[4],xmm14[5],xmm13[5],xmm14[6],xmm13[6],xmm14[7],xmm13[7]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm17, %zmm2
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm1, %zmm2 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm2, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vmovdqa 64(%r10), %ymm6
@@ -8522,47 +8525,47 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm2 = ymm5[0],ymm6[0],ymm5[1],ymm6[1],ymm5[2],ymm6[2],ymm5[3],ymm6[3],ymm5[8],ymm6[8],ymm5[9],ymm6[9],ymm5[10],ymm6[10],ymm5[11],ymm6[11]
; AVX512DQ-FCP-NEXT: vmovdqa 64(%r9), %ymm3
; AVX512DQ-FCP-NEXT: vmovdqa 64(%r8), %ymm4
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm12 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm17, %zmm12
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm10 = ymm4[0],ymm3[0],ymm4[1],ymm3[1],ymm4[2],ymm3[2],ymm4[3],ymm3[3],ymm4[8],ymm3[8],ymm4[9],ymm3[9],ymm4[10],ymm3[10],ymm4[11],ymm3[11]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm18, %zmm10
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rcx), %ymm2
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdx), %ymm13
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm15 = ymm13[0],ymm2[0],ymm13[1],ymm2[1],ymm13[2],ymm2[2],ymm13[3],ymm2[3],ymm13[8],ymm2[8],ymm13[9],ymm2[9],ymm13[10],ymm2[10],ymm13[11],ymm2[11]
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm14 = ymm13[0],ymm2[0],ymm13[1],ymm2[1],ymm13[2],ymm2[2],ymm13[3],ymm2[3],ymm13[8],ymm2[8],ymm13[9],ymm2[9],ymm13[10],ymm2[10],ymm13[11],ymm2[11]
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rsi), %ymm1
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %ymm0
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm0[0],ymm1[0],ymm0[1],ymm1[1],ymm0[2],ymm1[2],ymm0[3],ymm1[3],ymm0[8],ymm1[8],ymm0[9],ymm1[9],ymm0[10],ymm1[10],ymm0[11],ymm1[11]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm15, %zmm19, %zmm7
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm12, %zmm7 {%k1}
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm14, %zmm19, %zmm7
+; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm10, %zmm7 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm7, (%rsp) # 64-byte Spill
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm6[4],ymm5[5],ymm6[5],ymm5[6],ymm6[6],ymm5[7],ymm6[7],ymm5[12],ymm6[12],ymm5[13],ymm6[13],ymm5[14],ymm6[14],ymm5[15],ymm6[15]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm4[4],ymm3[4],ymm4[5],ymm3[5],ymm4[6],ymm3[6],ymm4[7],ymm3[7],ymm4[12],ymm3[12],ymm4[13],ymm3[13],ymm4[14],ymm3[14],ymm4[15],ymm3[15]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm5, %zmm17, %zmm3
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm5, %zmm18, %zmm3
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm13[4],ymm2[4],ymm13[5],ymm2[5],ymm13[6],ymm2[6],ymm13[7],ymm2[7],ymm13[12],ymm2[12],ymm13[13],ymm2[13],ymm13[14],ymm2[14],ymm13[15],ymm2[15]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm0 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm19, %zmm0
-; AVX512DQ-FCP-NEXT: vmovdqa 64(%r10), %xmm8
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, %zmm0 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm0, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
+; AVX512DQ-FCP-NEXT: vmovdqa 64(%r10), %xmm7
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rax), %xmm2
-; AVX512DQ-FCP-NEXT: vmovdqa 64(%r9), %xmm7
+; AVX512DQ-FCP-NEXT: vmovdqa 64(%r9), %xmm8
; AVX512DQ-FCP-NEXT: vmovdqa 64(%r8), %xmm3
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm8[0],xmm2[1],xmm8[1],xmm2[2],xmm8[2],xmm2[3],xmm8[3]
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm7[0],xmm3[1],xmm7[1],xmm3[2],xmm7[2],xmm3[3],xmm7[3]
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm4 = xmm2[0],xmm7[0],xmm2[1],xmm7[1],xmm2[2],xmm7[2],xmm2[3],xmm7[3]
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm3[0],xmm8[0],xmm3[1],xmm8[1],xmm3[2],xmm8[2],xmm3[3],xmm8[3]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm4, %zmm16, %zmm5
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rcx), %xmm4
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdx), %xmm6
-; AVX512DQ-FCP-NEXT: vmovdqa 64(%rsi), %xmm13
+; AVX512DQ-FCP-NEXT: vmovdqa 64(%rsi), %xmm14
; AVX512DQ-FCP-NEXT: vmovdqa 64(%rdi), %xmm0
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm6[0],xmm4[0],xmm6[1],xmm4[1],xmm6[2],xmm4[2],xmm6[3],xmm4[3]
-; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm13[0],xmm0[1],xmm13[1],xmm0[2],xmm13[2],xmm0[3],xmm13[3]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm18, %zmm9
+; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm9 = xmm0[0],xmm14[0],xmm0[1],xmm14[1],xmm0[2],xmm14[2],xmm0[3],xmm14[3]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm9
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm9 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqu64 %zmm9, {{[-0-9]+}}(%r{{[sb]}}p) # 64-byte Spill
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm8[4],xmm2[5],xmm8[5],xmm2[6],xmm8[6],xmm2[7],xmm8[7]
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm7[4],xmm3[5],xmm7[5],xmm3[6],xmm7[6],xmm3[7],xmm7[7]
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm2[4],xmm7[4],xmm2[5],xmm7[5],xmm2[6],xmm7[6],xmm2[7],xmm7[7]
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm2 = xmm3[4],xmm8[4],xmm3[5],xmm8[5],xmm3[6],xmm8[6],xmm3[7],xmm8[7]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm16, %zmm2
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm6[4],xmm4[4],xmm6[5],xmm4[5],xmm6[6],xmm4[6],xmm6[7],xmm4[7]
-; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm13[4],xmm0[5],xmm13[5],xmm0[6],xmm13[6],xmm0[7],xmm13[7]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm18, %zmm15
+; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm15 = xmm0[4],xmm14[4],xmm0[5],xmm14[5],xmm0[6],xmm14[6],xmm0[7],xmm14[7]
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm15
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm2, %zmm15 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r10), %ymm8
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rax), %ymm4
@@ -8570,7 +8573,7 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r9), %ymm3
; AVX512DQ-FCP-NEXT: vmovdqa 32(%r8), %ymm5
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm6 = ymm5[0],ymm3[0],ymm5[1],ymm3[1],ymm5[2],ymm3[2],ymm5[3],ymm3[3],ymm5[8],ymm3[8],ymm5[9],ymm3[9],ymm5[10],ymm3[10],ymm5[11],ymm3[11]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm17, %zmm6
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm18, %zmm6
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rcx), %ymm2
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdx), %ymm7
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm9 = ymm7[0],ymm2[0],ymm7[1],ymm2[1],ymm7[2],ymm2[2],ymm7[3],ymm2[3],ymm7[8],ymm2[8],ymm7[9],ymm2[9],ymm7[10],ymm2[10],ymm7[11],ymm2[11]
@@ -8581,7 +8584,7 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, %zmm14 {%k1}
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm4 = ymm4[4],ymm8[4],ymm4[5],ymm8[5],ymm4[6],ymm8[6],ymm4[7],ymm8[7],ymm4[12],ymm8[12],ymm4[13],ymm8[13],ymm4[14],ymm8[14],ymm4[15],ymm8[15]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm5[4],ymm3[4],ymm5[5],ymm3[5],ymm5[6],ymm3[6],ymm5[7],ymm3[7],ymm5[12],ymm3[12],ymm5[13],ymm3[13],ymm5[14],ymm3[14],ymm5[15],ymm3[15]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm4, %zmm17, %zmm5
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm4, %zmm18, %zmm5
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm2 = ymm7[4],ymm2[4],ymm7[5],ymm2[5],ymm7[6],ymm2[6],ymm7[7],ymm2[7],ymm7[12],ymm2[12],ymm7[13],ymm2[13],ymm7[14],ymm2[14],ymm7[15],ymm2[15]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm13 = ymm0[4],ymm1[4],ymm0[5],ymm1[5],ymm0[6],ymm1[6],ymm0[7],ymm1[7],ymm0[12],ymm1[12],ymm0[13],ymm1[13],ymm0[14],ymm1[14],ymm0[15],ymm1[15]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm19, %zmm13
@@ -8599,14 +8602,14 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa 32(%rdi), %xmm0
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm8[0],xmm7[0],xmm8[1],xmm7[1],xmm8[2],xmm7[2],xmm8[3],xmm7[3]
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm12 = xmm0[0],xmm9[0],xmm0[1],xmm9[1],xmm0[2],xmm9[2],xmm0[3],xmm9[3]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm18, %zmm12
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm12
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm6, %zmm12 {%k1}
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm3[4],xmm10[4],xmm3[5],xmm10[5],xmm3[6],xmm10[6],xmm3[7],xmm10[7]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm3 = xmm5[4],xmm4[4],xmm5[5],xmm4[5],xmm5[6],xmm4[6],xmm5[7],xmm4[7]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm16, %zmm3
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm8[4],xmm7[4],xmm8[5],xmm7[5],xmm8[6],xmm7[6],xmm8[7],xmm7[7]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} xmm4 = xmm0[4],xmm9[4],xmm0[5],xmm9[5],xmm0[6],xmm9[6],xmm0[7],xmm9[7]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm18, %zmm4
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm1, %zmm17, %zmm4
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, %zmm4 {%k1}
; AVX512DQ-FCP-NEXT: vmovdqa (%r10), %ymm10
; AVX512DQ-FCP-NEXT: vmovdqa (%rax), %ymm3
@@ -8614,7 +8617,7 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa (%r9), %ymm5
; AVX512DQ-FCP-NEXT: vmovdqa (%r8), %ymm6
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm7 = ymm6[0],ymm5[0],ymm6[1],ymm5[1],ymm6[2],ymm5[2],ymm6[3],ymm5[3],ymm6[8],ymm5[8],ymm6[9],ymm5[9],ymm6[10],ymm5[10],ymm6[11],ymm5[11]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm17, %zmm7
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm0, %zmm18, %zmm7
; AVX512DQ-FCP-NEXT: vmovdqa (%rcx), %ymm8
; AVX512DQ-FCP-NEXT: vmovdqa (%rdx), %ymm9
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} ymm11 = ymm9[0],ymm8[0],ymm9[1],ymm8[1],ymm9[2],ymm8[2],ymm9[3],ymm8[3],ymm9[8],ymm8[8],ymm9[9],ymm8[9],ymm9[10],ymm8[10],ymm9[11],ymm8[11]
@@ -8625,25 +8628,25 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm7, %zmm0 {%k1}
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm3[4],ymm10[4],ymm3[5],ymm10[5],ymm3[6],ymm10[6],ymm3[7],ymm10[7],ymm3[12],ymm10[12],ymm3[13],ymm10[13],ymm3[14],ymm10[14],ymm3[15],ymm10[15]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm5 = ymm6[4],ymm5[4],ymm6[5],ymm5[5],ymm6[6],ymm5[6],ymm6[7],ymm5[7],ymm6[12],ymm5[12],ymm6[13],ymm5[13],ymm6[14],ymm5[14],ymm6[15],ymm5[15]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm17, %zmm5
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm18, %zmm5
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm3 = ymm9[4],ymm8[4],ymm9[5],ymm8[5],ymm9[6],ymm8[6],ymm9[7],ymm8[7],ymm9[12],ymm8[12],ymm9[13],ymm8[13],ymm9[14],ymm8[14],ymm9[15],ymm8[15]
; AVX512DQ-FCP-NEXT: vpunpckhwd {{.*#+}} ymm1 = ymm1[4],ymm2[4],ymm1[5],ymm2[5],ymm1[6],ymm2[6],ymm1[7],ymm2[7],ymm1[12],ymm2[12],ymm1[13],ymm2[13],ymm1[14],ymm2[14],ymm1[15],ymm2[15]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm3, %zmm19, %zmm1
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, %zmm1 {%k1}
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm21, %xmm2
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm20, %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm21, %xmm3
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm3[0],xmm2[1],xmm3[1],xmm2[2],xmm3[2],xmm2[3],xmm3[3]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm23, %xmm3
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm24, %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm22, %xmm3
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm23, %xmm5
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm3 = xmm3[0],xmm5[0],xmm3[1],xmm5[1],xmm3[2],xmm5[2],xmm3[3],xmm5[3]
; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm16, %zmm3
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm25, %xmm2
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm26, %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm24, %xmm2
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm25, %xmm5
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm2 = xmm2[0],xmm5[0],xmm2[1],xmm5[1],xmm2[2],xmm5[2],xmm2[3],xmm5[3]
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm27, %xmm5
-; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm28, %xmm6
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm26, %xmm5
+; AVX512DQ-FCP-NEXT: vmovdqa64 %xmm27, %xmm6
; AVX512DQ-FCP-NEXT: vpunpcklwd {{.*#+}} xmm5 = xmm5[0],xmm6[0],xmm5[1],xmm6[1],xmm5[2],xmm6[2],xmm5[3],xmm6[3]
-; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm18, %zmm5
+; AVX512DQ-FCP-NEXT: vpermt2d %zmm2, %zmm17, %zmm5
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm3, %zmm5 {%k1}
; AVX512DQ-FCP-NEXT: movq {{[0-9]+}}(%rsp), %rax
; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm5, (%rax)
@@ -8668,8 +8671,9 @@ define void @store_i16_stride8_vf64(ptr %in.vecptr0, ptr %in.vecptr1, ptr %in.ve
; AVX512DQ-FCP-NEXT: vmovaps %zmm0, 960(%rax)
; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
; AVX512DQ-FCP-NEXT: vmovaps %zmm0, 896(%rax)
-; AVX512DQ-FCP-NEXT: vmovdqa64 %zmm20, 64(%rax)
-; AVX512DQ-FCP-NEXT: addq $328, %rsp # imm = 0x148
+; AVX512DQ-FCP-NEXT: vmovups {{[-0-9]+}}(%r{{[sb]}}p), %zmm0 # 64-byte Reload
+; AVX512DQ-FCP-NEXT: vmovaps %zmm0, 64(%rax)
+; AVX512DQ-FCP-NEXT: addq $392, %rsp # imm = 0x188
; AVX512DQ-FCP-NEXT: vzeroupper
; AVX512DQ-FCP-NEXT: retq
;
diff --git a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
index dbce67dc9127ec..340dbd3e6aceba 100644
--- a/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
+++ b/llvm/test/CodeGen/X86/vector-shuffle-256-v16.ll
@@ -4351,8 +4351,8 @@ define <16 x i16> @shuffle_v16i16_06_06_07_05_01_06_04_11_14_14_15_13_09_14_12_1
;
; AVX2-SLOW-LABEL: shuffle_v16i16_06_06_07_05_01_06_04_11_14_14_15_13_09_14_12_11:
; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,5,2,3,4,5,6,7]
-; AVX2-SLOW-NEXT: vpermd %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,2,3]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7]
; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[12,13,12,13,14,15,10,11,2,3,12,13,8,9,6,7,28,29,28,29,30,31,26,27,18,19,28,29,24,25,22,23]
; AVX2-SLOW-NEXT: retq
;
@@ -4365,8 +4365,8 @@ define <16 x i16> @shuffle_v16i16_06_06_07_05_01_06_04_11_14_14_15_13_09_14_12_1
;
; AVX2-FAST-PERLANE-LABEL: shuffle_v16i16_06_06_07_05_01_06_04_11_14_14_15_13_09_14_12_11:
; AVX2-FAST-PERLANE: # %bb.0:
-; AVX2-FAST-PERLANE-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,5,2,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT: vpermd %ymm0, %ymm1, %ymm0
+; AVX2-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,2,3]
+; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7]
; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[12,13,12,13,14,15,10,11,2,3,12,13,8,9,6,7,28,29,28,29,30,31,26,27,18,19,28,29,24,25,22,23]
; AVX2-FAST-PERLANE-NEXT: retq
;
@@ -4386,8 +4386,8 @@ define <16 x i16> @shuffle_v16i16_06_06_07_05_01_06_04_11_14_14_15_13_09_14_12_1
;
; XOPAVX2-LABEL: shuffle_v16i16_06_06_07_05_01_06_04_11_14_14_15_13_09_14_12_11:
; XOPAVX2: # %bb.0:
-; XOPAVX2-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,5,2,3,4,5,6,7]
-; XOPAVX2-NEXT: vpermd %ymm0, %ymm1, %ymm0
+; XOPAVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,2,3]
+; XOPAVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2,3,4],ymm1[5],ymm0[6,7]
; XOPAVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[12,13,12,13,14,15,10,11,2,3,12,13,8,9,6,7,28,29,28,29,30,31,26,27,18,19,28,29,24,25,22,23]
; XOPAVX2-NEXT: retq
%shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 6, i32 6, i32 7, i32 5, i32 1, i32 6, i32 4, i32 11, i32 14, i32 14, i32 15, i32 13, i32 9, i32 14, i32 12, i32 11>
@@ -5097,8 +5097,8 @@ define <16 x i16> @shuffle_v16i16_03_07_01_00_02_07_03_13_11_15_09_08_10_15_11_1
;
; AVX2-SLOW-LABEL: shuffle_v16i16_03_07_01_00_02_07_03_13_11_15_09_08_10_15_11_13:
; AVX2-SLOW: # %bb.0:
-; AVX2-SLOW-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,1,6,3,4,5,6,7]
-; AVX2-SLOW-NEXT: vpermd %ymm0, %ymm1, %ymm0
+; AVX2-SLOW-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,2,3]
+; AVX2-SLOW-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4,5],ymm1[6],ymm0[7]
; AVX2-SLOW-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[6,7,14,15,2,3,0,1,4,5,14,15,6,7,10,11,22,23,30,31,18,19,16,17,20,21,30,31,22,23,26,27]
; AVX2-SLOW-NEXT: retq
;
@@ -5111,8 +5111,8 @@ define <16 x i16> @shuffle_v16i16_03_07_01_00_02_07_03_13_11_15_09_08_10_15_11_1
;
; AVX2-FAST-PERLANE-LABEL: shuffle_v16i16_03_07_01_00_02_07_03_13_11_15_09_08_10_15_11_13:
; AVX2-FAST-PERLANE: # %bb.0:
-; AVX2-FAST-PERLANE-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,1,6,3,4,5,6,7]
-; AVX2-FAST-PERLANE-NEXT: vpermd %ymm0, %ymm1, %ymm0
+; AVX2-FAST-PERLANE-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,2,3]
+; AVX2-FAST-PERLANE-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4,5],ymm1[6],ymm0[7]
; AVX2-FAST-PERLANE-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[6,7,14,15,2,3,0,1,4,5,14,15,6,7,10,11,22,23,30,31,18,19,16,17,20,21,30,31,22,23,26,27]
; AVX2-FAST-PERLANE-NEXT: retq
;
@@ -5132,8 +5132,8 @@ define <16 x i16> @shuffle_v16i16_03_07_01_00_02_07_03_13_11_15_09_08_10_15_11_1
;
; XOPAVX2-LABEL: shuffle_v16i16_03_07_01_00_02_07_03_13_11_15_09_08_10_15_11_13:
; XOPAVX2: # %bb.0:
-; XOPAVX2-NEXT: vpmovsxbd {{.*#+}} ymm1 = [0,1,6,3,4,5,6,7]
-; XOPAVX2-NEXT: vpermd %ymm0, %ymm1, %ymm0
+; XOPAVX2-NEXT: vpermq {{.*#+}} ymm1 = ymm0[2,3,2,3]
+; XOPAVX2-NEXT: vpblendd {{.*#+}} ymm0 = ymm0[0,1],ymm1[2],ymm0[3,4,5],ymm1[6],ymm0[7]
; XOPAVX2-NEXT: vpshufb {{.*#+}} ymm0 = ymm0[6,7,14,15,2,3,0,1,4,5,14,15,6,7,10,11,22,23,30,31,18,19,16,17,20,21,30,31,22,23,26,27]
; XOPAVX2-NEXT: retq
%shuffle = shufflevector <16 x i16> %a, <16 x i16> %b, <16 x i32> <i32 3, i32 7, i32 1, i32 0, i32 2, i32 7, i32 3, i32 13, i32 11, i32 15, i32 9, i32 8, i32 10, i32 15, i32 11, i32 13>
diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
index 6489f950482b6a..28c00a47dda0ec 100644
--- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
+++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll
@@ -3594,73 +3594,39 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i
; AVX2-NEXT: vzeroupper
; AVX2-NEXT: retq
;
-; AVX512F-SLOW-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
-; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512F-SLOW-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0
-; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
-; AVX512F-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512F-SLOW-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512F-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
-; AVX512F-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
-; AVX512F-SLOW-NEXT: vmovdqa %ymm1, (%rcx)
-; AVX512F-SLOW-NEXT: vmovdqa %ymm0, 32(%rcx)
-; AVX512F-SLOW-NEXT: vzeroupper
-; AVX512F-SLOW-NEXT: retq
-;
-; AVX512F-FAST-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
-; AVX512F-FAST: # %bb.0:
-; AVX512F-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512F-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512F-FAST-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-FAST-NEXT: vpbroadcastw %xmm0, %ymm2
-; AVX512F-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7],ymm2[8],ymm1[9],ymm2[10],ymm1[11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
-; AVX512F-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512F-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero
-; AVX512F-FAST-NEXT: vpaddb (%rdx), %ymm1, %ymm1
-; AVX512F-FAST-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
-; AVX512F-FAST-NEXT: vmovdqa %ymm0, 32(%rcx)
-; AVX512F-FAST-NEXT: vmovdqa %ymm1, (%rcx)
-; AVX512F-FAST-NEXT: vzeroupper
-; AVX512F-FAST-NEXT: retq
-;
-; AVX512DQ-SLOW-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
-; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512DQ-SLOW-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
-; AVX512DQ-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512DQ-SLOW-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
-; AVX512DQ-SLOW-NEXT: vpaddb (%rdx), %ymm1, %ymm1
-; AVX512DQ-SLOW-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm1, (%rcx)
-; AVX512DQ-SLOW-NEXT: vmovdqa %ymm0, 32(%rcx)
-; AVX512DQ-SLOW-NEXT: vzeroupper
-; AVX512DQ-SLOW-NEXT: retq
+; AVX512F-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
+; AVX512F: # %bb.0:
+; AVX512F-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512F-NEXT: vmovdqa 48(%rdi), %xmm1
+; AVX512F-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512F-NEXT: vpaddb (%rsi), %xmm0, %xmm0
+; AVX512F-NEXT: vpbroadcastw %xmm0, %ymm0
+; AVX512F-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
+; AVX512F-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512F-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX512F-NEXT: vpaddb (%rdx), %ymm1, %ymm1
+; AVX512F-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
+; AVX512F-NEXT: vmovdqa %ymm1, (%rcx)
+; AVX512F-NEXT: vmovdqa %ymm0, 32(%rcx)
+; AVX512F-NEXT: vzeroupper
+; AVX512F-NEXT: retq
;
-; AVX512DQ-FAST-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
-; AVX512DQ-FAST: # %bb.0:
-; AVX512DQ-FAST-NEXT: vmovdqa (%rdi), %ymm0
-; AVX512DQ-FAST-NEXT: vpaddb (%rsi), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa 48(%rdi), %xmm1
-; AVX512DQ-FAST-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-FAST-NEXT: vpbroadcastw %xmm0, %ymm2
-; AVX512DQ-FAST-NEXT: vpblendw {{.*#+}} ymm1 = ymm2[0],ymm1[1],ymm2[2],ymm1[3],ymm2[4],ymm1[5],ymm2[6],ymm1[7],ymm2[8],ymm1[9],ymm2[10],ymm1[11],ymm2[12],ymm1[13],ymm2[14],ymm1[15]
-; AVX512DQ-FAST-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
-; AVX512DQ-FAST-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero
-; AVX512DQ-FAST-NEXT: vpaddb (%rdx), %ymm1, %ymm1
-; AVX512DQ-FAST-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm0, 32(%rcx)
-; AVX512DQ-FAST-NEXT: vmovdqa %ymm1, (%rcx)
-; AVX512DQ-FAST-NEXT: vzeroupper
-; AVX512DQ-FAST-NEXT: retq
+; AVX512DQ-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
+; AVX512DQ: # %bb.0:
+; AVX512DQ-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512DQ-NEXT: vmovdqa 48(%rdi), %xmm1
+; AVX512DQ-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
+; AVX512DQ-NEXT: vpaddb (%rsi), %xmm0, %xmm0
+; AVX512DQ-NEXT: vpbroadcastw %xmm0, %ymm0
+; AVX512DQ-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1],ymm0[2],ymm1[3],ymm0[4],ymm1[5],ymm0[6],ymm1[7],ymm0[8],ymm1[9],ymm0[10],ymm1[11],ymm0[12],ymm1[13],ymm0[14],ymm1[15]
+; AVX512DQ-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
+; AVX512DQ-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero
+; AVX512DQ-NEXT: vpaddb (%rdx), %ymm1, %ymm1
+; AVX512DQ-NEXT: vpaddb 32(%rdx), %ymm0, %ymm0
+; AVX512DQ-NEXT: vmovdqa %ymm1, (%rcx)
+; AVX512DQ-NEXT: vmovdqa %ymm0, 32(%rcx)
+; AVX512DQ-NEXT: vzeroupper
+; AVX512DQ-NEXT: retq
;
; AVX512BW-SLOW-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12:
; AVX512BW-SLOW: # %bb.0:
@@ -4013,10 +3979,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
;
; AVX512F-SLOW-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
; AVX512F-SLOW: # %bb.0:
-; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512F-SLOW-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512F-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512F-SLOW-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512F-SLOW-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512F-SLOW-NEXT: vpaddb (%rsi), %xmm0, %xmm0
; AVX512F-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX512F-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]
; AVX512F-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
@@ -4048,10 +4014,10 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in.
;
; AVX512DQ-SLOW-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6:
; AVX512DQ-SLOW: # %bb.0:
-; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %xmm0
+; AVX512DQ-SLOW-NEXT: vmovdqa (%rdi), %ymm0
+; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %ymm0, %ymm0
; AVX512DQ-SLOW-NEXT: vmovdqa 48(%rdi), %xmm1
; AVX512DQ-SLOW-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1
-; AVX512DQ-SLOW-NEXT: vpaddb (%rsi), %xmm0, %xmm0
; AVX512DQ-SLOW-NEXT: vpbroadcastw %xmm0, %ymm0
; AVX512DQ-SLOW-NEXT: vpblendw {{.*#+}} ymm1 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7],ymm0[8],ymm1[9,10,11],ymm0[12],ymm1[13,14,15]
; AVX512DQ-SLOW-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %ymm1, %ymm1
More information about the llvm-commits
mailing list