[llvm] [AMDGPU] Don't cluster DS-instrs together (PR #180908)
via llvm-commits
llvm-commits at lists.llvm.org
Sun Feb 15 06:29:46 PST 2026
https://github.com/lijinpei-amd updated https://github.com/llvm/llvm-project/pull/180908
>From 9826fede4d04af695957a076ebba57fcb2206d57 Mon Sep 17 00:00:00 2001
From: Li Jinpei <jinpli at amd.com>
Date: Tue, 10 Feb 2026 20:47:06 +0800
Subject: [PATCH] [AMDGPU] Don't cluster lds instrs together
When scheduling, LLVM tends to emit load/store instrs continuously.
However, for GPUs, there are no architectural benefits in doing so for
lds/shared-memory accesses. In fact, trying to schedule lds instrs
together may lead to less optimal schedule results. This commit undoes
previous clustering of lds instrs for AMDGPU in two places:
- Remove creation of cluster edge for lds instrs in schedule DAG
mutation.
- Remove bundling of lds instrs in SIPostRABundler.
---
llvm/lib/Target/AMDGPU/SIInstrInfo.cpp | 12 +
llvm/lib/Target/AMDGPU/SIPostRABundler.cpp | 6 +-
.../AMDGPU/GlobalISel/load-local.128.ll | 178 +-
.../AMDGPU/GlobalISel/load-local.96.ll | 125 +-
.../AMDGPU/GlobalISel/load-unaligned.ll | 158 +-
.../AMDGPU/GlobalISel/store-local.128.ll | 131 +-
.../AMDGPU/GlobalISel/store-local.96.ll | 67 +-
.../test/CodeGen/AMDGPU/atomic_store_local.ll | 16 +-
llvm/test/CodeGen/AMDGPU/ds-alignment.ll | 194 +-
llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll | 82 +-
llvm/test/CodeGen/AMDGPU/ds_read2.ll | 201 +-
llvm/test/CodeGen/AMDGPU/ds_write2.ll | 50 +-
llvm/test/CodeGen/AMDGPU/ds_write2_a_v.ll | 16 +-
.../AMDGPU/expand-waitcnt-profiling.ll | 32 +-
.../CodeGen/AMDGPU/extract-subvector-16bit.ll | 16 +-
llvm/test/CodeGen/AMDGPU/freeze.ll | 86 +-
llvm/test/CodeGen/AMDGPU/lds-frame-extern.ll | 16 +-
.../test/CodeGen/AMDGPU/lds-misaligned-bug.ll | 25 +-
llvm/test/CodeGen/AMDGPU/load-lo16.ll | 16 +-
llvm/test/CodeGen/AMDGPU/load-local-i16.ll | 3302 +++++++++--------
.../AMDGPU/load-local-redundant-copies.ll | 69 +-
llvm/test/CodeGen/AMDGPU/load-local.128.ll | 251 +-
llvm/test/CodeGen/AMDGPU/load-local.96.ll | 167 +-
.../CodeGen/AMDGPU/local-memory.amdgcn.ll | 2 +-
.../AMDGPU/lower-lds-struct-aa-memcpy.ll | 6 +-
.../CodeGen/AMDGPU/lower-lds-struct-aa.ll | 20 +-
llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll | 12 +-
.../CodeGen/AMDGPU/memintrinsic-unroll.ll | 6 +-
.../AMDGPU/memmove-param-combinations.ll | 50 +-
.../AMDGPU/memset-param-combinations.ll | 12 +-
.../CodeGen/AMDGPU/misched-remat-revert.ll | 4 +-
.../AMDGPU/module-lds-false-sharing.ll | 4 +-
llvm/test/CodeGen/AMDGPU/packed-op-sel.ll | 2 +-
.../CodeGen/AMDGPU/postra-bundle-memops.mir | 18 +-
...schedule-regpressure-ilp-metric-spills.mir | 7 +-
.../AMDGPU/schedule-regpressure-limit2.ll | 4 +-
.../AMDGPU/schedule-relaxed-occupancy.ll | 14 +-
llvm/test/CodeGen/AMDGPU/store-local.128.ll | 106 +-
llvm/test/CodeGen/AMDGPU/store-local.96.ll | 49 +-
llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll | 22 +-
40 files changed, 2853 insertions(+), 2701 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
index 24aa31a318df3..fb53490d24e75 100644
--- a/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
+++ b/llvm/lib/Target/AMDGPU/SIInstrInfo.cpp
@@ -563,12 +563,24 @@ static bool memOpsHaveSameBasePtr(const MachineInstr &MI1,
return Base1 == Base2;
}
+static bool isDSInstrOperands(ArrayRef<const MachineOperand *> operands) {
+ if (operands.empty()) {
+ return false;
+ }
+ const auto *MI = operands.front()->getParent();
+ return SIInstrInfo::isDS(*MI);
+}
+
bool SIInstrInfo::shouldClusterMemOps(ArrayRef<const MachineOperand *> BaseOps1,
int64_t Offset1, bool OffsetIsScalable1,
ArrayRef<const MachineOperand *> BaseOps2,
int64_t Offset2, bool OffsetIsScalable2,
unsigned ClusterSize,
unsigned NumBytes) const {
+ if (isDSInstrOperands(BaseOps1) || isDSInstrOperands(BaseOps2)) {
+ return false;
+ }
+
// If the mem ops (to be clustered) do not have the same base ptr, then they
// should not be clustered
unsigned MaxMemoryClusterDWords = DefaultMemoryClusterDWordsLimit;
diff --git a/llvm/lib/Target/AMDGPU/SIPostRABundler.cpp b/llvm/lib/Target/AMDGPU/SIPostRABundler.cpp
index 787f7b30458dd..24c0867e197b5 100644
--- a/llvm/lib/Target/AMDGPU/SIPostRABundler.cpp
+++ b/llvm/lib/Target/AMDGPU/SIPostRABundler.cpp
@@ -61,9 +61,9 @@ class SIPostRABundler {
};
constexpr uint64_t MemFlags = SIInstrFlags::MTBUF | SIInstrFlags::MUBUF |
- SIInstrFlags::SMRD | SIInstrFlags::DS |
- SIInstrFlags::FLAT | SIInstrFlags::MIMG |
- SIInstrFlags::VIMAGE | SIInstrFlags::VSAMPLE;
+ SIInstrFlags::SMRD | SIInstrFlags::FLAT |
+ SIInstrFlags::MIMG | SIInstrFlags::VIMAGE |
+ SIInstrFlags::VSAMPLE;
} // End anonymous namespace.
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
index d7fcbd5d623c9..1933621eadb0a 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.128.ll
@@ -46,44 +46,48 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_u8 v1, v0
; GFX9-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX9-NEXT: ds_read_u8 v3, v0 offset:2
-; GFX9-NEXT: ds_read_u8 v4, v0 offset:3
+; GFX9-NEXT: ds_read_u8 v3, v0 offset:3
+; GFX9-NEXT: ds_read_u8 v4, v0 offset:2
; GFX9-NEXT: ds_read_u8 v5, v0 offset:4
-; GFX9-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX9-NEXT: ds_read_u8 v7, v0 offset:6
-; GFX9-NEXT: ds_read_u8 v8, v0 offset:7
-; GFX9-NEXT: s_waitcnt lgkmcnt(6)
+; GFX9-NEXT: ds_read_u8 v6, v0 offset:9
+; GFX9-NEXT: s_waitcnt lgkmcnt(4)
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 8, v1
+; GFX9-NEXT: ds_read_u8 v2, v0 offset:5
; GFX9-NEXT: s_waitcnt lgkmcnt(4)
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 24, v4
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX9-NEXT: s_waitcnt lgkmcnt(3)
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or3_b32 v4, v3, v4, v1
+; GFX9-NEXT: ds_read_u8 v1, v0 offset:7
+; GFX9-NEXT: ds_read_u8 v3, v0 offset:6
+; GFX9-NEXT: s_waitcnt lgkmcnt(2)
+; GFX9-NEXT: v_lshl_or_b32 v2, v2, 8, v5
+; GFX9-NEXT: ds_read_u8 v5, v0 offset:8
+; GFX9-NEXT: ds_read_u8 v7, v0 offset:11
+; GFX9-NEXT: s_waitcnt lgkmcnt(3)
+; GFX9-NEXT: v_lshlrev_b32_e32 v1, 24, v1
+; GFX9-NEXT: s_waitcnt lgkmcnt(2)
; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_or3_b32 v4, v2, v3, v1
+; GFX9-NEXT: v_or3_b32 v1, v1, v3, v2
+; GFX9-NEXT: ds_read_u8 v3, v0 offset:10
; GFX9-NEXT: s_waitcnt lgkmcnt(2)
-; GFX9-NEXT: v_lshl_or_b32 v1, v6, 8, v5
-; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 24, v8
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v7
-; GFX9-NEXT: v_or3_b32 v1, v2, v3, v1
-; GFX9-NEXT: ds_read_u8 v2, v0 offset:8
-; GFX9-NEXT: ds_read_u8 v3, v0 offset:9
-; GFX9-NEXT: ds_read_u8 v5, v0 offset:10
-; GFX9-NEXT: ds_read_u8 v6, v0 offset:11
-; GFX9-NEXT: ds_read_u8 v7, v0 offset:12
-; GFX9-NEXT: ds_read_u8 v8, v0 offset:13
-; GFX9-NEXT: ds_read_u8 v9, v0 offset:14
-; GFX9-NEXT: ds_read_u8 v0, v0 offset:15
-; GFX9-NEXT: s_waitcnt lgkmcnt(6)
-; GFX9-NEXT: v_lshl_or_b32 v2, v3, 8, v2
+; GFX9-NEXT: v_lshl_or_b32 v2, v6, 8, v5
+; GFX9-NEXT: s_waitcnt lgkmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 24, v7
+; GFX9-NEXT: ds_read_u8 v6, v0 offset:12
+; GFX9-NEXT: ds_read_u8 v7, v0 offset:13
+; GFX9-NEXT: ds_read_u8 v8, v0 offset:15
+; GFX9-NEXT: ds_read_u8 v0, v0 offset:14
; GFX9-NEXT: s_waitcnt lgkmcnt(4)
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 24, v6
-; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v5
-; GFX9-NEXT: v_or3_b32 v2, v3, v5, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX9-NEXT: v_or3_b32 v2, v5, v3, v2
; GFX9-NEXT: s_waitcnt lgkmcnt(2)
-; GFX9-NEXT: v_lshl_or_b32 v3, v8, 8, v7
+; GFX9-NEXT: v_lshl_or_b32 v3, v7, 8, v6
+; GFX9-NEXT: s_waitcnt lgkmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v5, 24, v8
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; GFX9-NEXT: v_lshlrev_b32_e32 v5, 16, v9
-; GFX9-NEXT: v_or3_b32 v3, v0, v5, v3
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or3_b32 v3, v5, v0, v3
; GFX9-NEXT: v_mov_b32_e32 v0, v4
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -91,53 +95,61 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_u8 v1, v0
-; GFX7-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
-; GFX7-NEXT: ds_read_u8 v4, v0 offset:3
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
-; GFX7-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
-; GFX7-NEXT: ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT: ds_read_u8 v1, v0 offset:1
+; GFX7-NEXT: ds_read_u8 v2, v0
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:3
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:2
+; GFX7-NEXT: ds_read_u8 v6, v0 offset:7
; GFX7-NEXT: s_waitcnt lgkmcnt(4)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:5
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
; GFX7-NEXT: v_or_b32_e32 v4, v2, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v6
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v6
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v3
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: ds_read_u8 v2, v0 offset:8
; GFX7-NEXT: ds_read_u8 v3, v0 offset:9
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:10
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:8
; GFX7-NEXT: ds_read_u8 v6, v0 offset:11
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:12
-; GFX7-NEXT: ds_read_u8 v8, v0 offset:13
-; GFX7-NEXT: ds_read_u8 v9, v0 offset:14
-; GFX7-NEXT: ds_read_u8 v0, v0 offset:15
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:10
+; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v3
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 24, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX7-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:13
+; GFX7-NEXT: ds_read_u8 v6, v0 offset:12
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:15
+; GFX7-NEXT: ds_read_u8 v0, v0 offset:14
; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v5
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v8
+; GFX7-NEXT: v_or_b32_e32 v3, v3, v6
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 24, v7
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v9
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v5, v0
; GFX7-NEXT: v_or_b32_e32 v3, v0, v3
; GFX7-NEXT: v_mov_b32_e32 v0, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -270,26 +282,30 @@ define <4 x i32> @load_lds_v4i32_align2(ptr addrspace(3) %ptr) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_u16 v1, v0
-; GFX7-NEXT: ds_read_u16 v2, v0 offset:2
-; GFX7-NEXT: ds_read_u16 v3, v0 offset:4
-; GFX7-NEXT: ds_read_u16 v4, v0 offset:6
-; GFX7-NEXT: ds_read_u16 v5, v0 offset:8
-; GFX7-NEXT: ds_read_u16 v6, v0 offset:10
-; GFX7-NEXT: ds_read_u16 v7, v0 offset:12
-; GFX7-NEXT: ds_read_u16 v8, v0 offset:14
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
+; GFX7-NEXT: ds_read_u16 v1, v0 offset:2
+; GFX7-NEXT: ds_read_u16 v2, v0
+; GFX7-NEXT: ds_read_u16 v3, v0 offset:6
+; GFX7-NEXT: ds_read_u16 v6, v0 offset:14
+; GFX7-NEXT: ds_read_u16 v5, v0 offset:4
; GFX7-NEXT: s_waitcnt lgkmcnt(4)
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_or_b32_e32 v4, v1, v2
+; GFX7-NEXT: ds_read_u16 v2, v0 offset:10
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v3
+; GFX7-NEXT: ds_read_u16 v3, v0 offset:8
+; GFX7-NEXT: ds_read_u16 v0, v0 offset:12
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v8
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-NEXT: v_or_b32_e32 v3, v3, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_lds_v4i32_align2:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll
index 191f2e0670e15..d36111dfc31b5 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-local.96.ll
@@ -46,35 +46,37 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: ds_read_u8 v1, v0
; GFX9-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX9-NEXT: ds_read_u8 v3, v0 offset:2
-; GFX9-NEXT: ds_read_u8 v4, v0 offset:3
+; GFX9-NEXT: ds_read_u8 v3, v0 offset:3
+; GFX9-NEXT: ds_read_u8 v4, v0 offset:2
; GFX9-NEXT: ds_read_u8 v5, v0 offset:4
-; GFX9-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX9-NEXT: ds_read_u8 v7, v0 offset:6
-; GFX9-NEXT: ds_read_u8 v8, v0 offset:7
-; GFX9-NEXT: s_waitcnt lgkmcnt(6)
+; GFX9-NEXT: ds_read_u8 v6, v0 offset:7
+; GFX9-NEXT: s_waitcnt lgkmcnt(4)
; GFX9-NEXT: v_lshl_or_b32 v1, v2, 8, v1
+; GFX9-NEXT: ds_read_u8 v2, v0 offset:5
; GFX9-NEXT: s_waitcnt lgkmcnt(4)
-; GFX9-NEXT: v_lshlrev_b32_e32 v2, 24, v4
-; GFX9-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX9-NEXT: v_or3_b32 v3, v2, v3, v1
-; GFX9-NEXT: s_waitcnt lgkmcnt(2)
-; GFX9-NEXT: v_lshl_or_b32 v1, v6, 8, v5
-; GFX9-NEXT: ds_read_u8 v2, v0 offset:8
-; GFX9-NEXT: ds_read_u8 v4, v0 offset:9
-; GFX9-NEXT: ds_read_u8 v5, v0 offset:10
-; GFX9-NEXT: ds_read_u8 v0, v0 offset:11
+; GFX9-NEXT: v_lshlrev_b32_e32 v3, 24, v3
+; GFX9-NEXT: s_waitcnt lgkmcnt(3)
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v4
+; GFX9-NEXT: v_or3_b32 v3, v3, v4, v1
+; GFX9-NEXT: s_waitcnt lgkmcnt(1)
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 24, v6
+; GFX9-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NEXT: v_lshl_or_b32 v1, v2, 8, v5
+; GFX9-NEXT: ds_read_u8 v2, v0 offset:6
+; GFX9-NEXT: ds_read_u8 v5, v0 offset:8
+; GFX9-NEXT: ds_read_u8 v6, v0 offset:9
+; GFX9-NEXT: ds_read_u8 v7, v0 offset:11
+; GFX9-NEXT: ds_read_u8 v0, v0 offset:10
; GFX9-NEXT: s_waitcnt lgkmcnt(4)
-; GFX9-NEXT: v_lshlrev_b32_e32 v6, 24, v8
-; GFX9-NEXT: v_lshlrev_b32_e32 v7, 16, v7
+; GFX9-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX9-NEXT: v_or3_b32 v1, v4, v2, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(2)
-; GFX9-NEXT: v_lshl_or_b32 v2, v4, 8, v2
+; GFX9-NEXT: v_lshl_or_b32 v2, v6, 8, v5
; GFX9-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-NEXT: v_lshlrev_b32_e32 v4, 16, v5
+; GFX9-NEXT: v_lshlrev_b32_e32 v4, 24, v7
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; GFX9-NEXT: v_or3_b32 v1, v6, v7, v1
-; GFX9-NEXT: v_or3_b32 v2, v0, v4, v2
+; GFX9-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX9-NEXT: v_or3_b32 v2, v4, v0, v2
; GFX9-NEXT: v_mov_b32_e32 v0, v3
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
@@ -82,42 +84,46 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_u8 v1, v0
-; GFX7-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
-; GFX7-NEXT: ds_read_u8 v4, v0 offset:3
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
-; GFX7-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
-; GFX7-NEXT: ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT: ds_read_u8 v1, v0 offset:1
+; GFX7-NEXT: ds_read_u8 v2, v0
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:3
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:2
+; GFX7-NEXT: ds_read_u8 v6, v0 offset:7
; GFX7-NEXT: s_waitcnt lgkmcnt(4)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:5
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
; GFX7-NEXT: v_or_b32_e32 v3, v2, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v6
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v6
; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:8
-; GFX7-NEXT: ds_read_u8 v6, v0 offset:9
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:10
-; GFX7-NEXT: ds_read_u8 v0, v0 offset:11
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v8
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
; GFX7-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:9
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:8
+; GFX7-NEXT: ds_read_u8 v6, v0 offset:11
+; GFX7-NEXT: ds_read_u8 v0, v0 offset:10
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v6
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v7
; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 24, v6
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
; GFX7-NEXT: v_or_b32_e32 v2, v0, v2
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -224,21 +230,24 @@ define <3 x i32> @load_lds_v3i32_align2(ptr addrspace(3) %ptr) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_u16 v1, v0
-; GFX7-NEXT: ds_read_u16 v2, v0 offset:2
-; GFX7-NEXT: ds_read_u16 v3, v0 offset:4
+; GFX7-NEXT: ds_read_u16 v1, v0 offset:2
+; GFX7-NEXT: ds_read_u16 v2, v0
; GFX7-NEXT: ds_read_u16 v4, v0 offset:6
-; GFX7-NEXT: ds_read_u16 v5, v0 offset:8
-; GFX7-NEXT: ds_read_u16 v6, v0 offset:10
+; GFX7-NEXT: ds_read_u16 v5, v0 offset:10
+; GFX7-NEXT: ds_read_u16 v3, v0 offset:4
+; GFX7-NEXT: ds_read_u16 v6, v0 offset:8
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(4)
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v4
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v5
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v6
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v6
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: load_lds_v3i32_align2:
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll
index b1de0eff05d30..23208c30b6b57 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll
@@ -22,53 +22,61 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_u8 v1, v0
-; GFX7-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
-; GFX7-NEXT: ds_read_u8 v4, v0 offset:3
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
-; GFX7-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
-; GFX7-NEXT: ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT: ds_read_u8 v1, v0 offset:1
+; GFX7-NEXT: ds_read_u8 v2, v0
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:3
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:2
+; GFX7-NEXT: ds_read_u8 v6, v0 offset:7
; GFX7-NEXT: s_waitcnt lgkmcnt(4)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v4
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:5
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
; GFX7-NEXT: v_or_b32_e32 v4, v2, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v6
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v6
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v8
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v3
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: ds_read_u8 v2, v0 offset:8
; GFX7-NEXT: ds_read_u8 v3, v0 offset:9
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:10
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:8
; GFX7-NEXT: ds_read_u8 v6, v0 offset:11
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:12
-; GFX7-NEXT: ds_read_u8 v8, v0 offset:13
-; GFX7-NEXT: ds_read_u8 v9, v0 offset:14
-; GFX7-NEXT: ds_read_u8 v0, v0 offset:15
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:10
+; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v3
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 24, v6
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v7
; GFX7-NEXT: v_or_b32_e32 v3, v3, v5
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:13
+; GFX7-NEXT: ds_read_u8 v6, v0 offset:12
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:15
+; GFX7-NEXT: ds_read_u8 v0, v0 offset:14
; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v5
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v8
+; GFX7-NEXT: v_or_b32_e32 v3, v3, v6
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 24, v7
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 16, v9
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v7
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v5, v0
; GFX7-NEXT: v_or_b32_e32 v3, v0, v3
; GFX7-NEXT: v_mov_b32_e32 v0, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -104,42 +112,46 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX7: ; %bb.0:
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
-; GFX7-NEXT: ds_read_u8 v1, v0
-; GFX7-NEXT: ds_read_u8 v2, v0 offset:1
-; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
-; GFX7-NEXT: ds_read_u8 v4, v0 offset:3
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
-; GFX7-NEXT: ds_read_u8 v6, v0 offset:5
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
-; GFX7-NEXT: ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT: s_waitcnt lgkmcnt(6)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v2
-; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT: ds_read_u8 v1, v0 offset:1
+; GFX7-NEXT: ds_read_u8 v2, v0
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:3
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:2
+; GFX7-NEXT: ds_read_u8 v6, v0 offset:7
; GFX7-NEXT: s_waitcnt lgkmcnt(4)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v3
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:5
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
; GFX7-NEXT: v_or_b32_e32 v2, v2, v3
; GFX7-NEXT: v_or_b32_e32 v3, v2, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v6
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v6
; GFX7-NEXT: s_waitcnt lgkmcnt(1)
; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:8
-; GFX7-NEXT: ds_read_u8 v6, v0 offset:9
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:10
-; GFX7-NEXT: ds_read_u8 v0, v0 offset:11
-; GFX7-NEXT: s_waitcnt lgkmcnt(4)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 24, v8
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
; GFX7-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:9
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:8
+; GFX7-NEXT: ds_read_u8 v6, v0 offset:11
+; GFX7-NEXT: ds_read_u8 v0, v0 offset:10
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v6
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 24, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v7
; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 24, v6
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
; GFX7-NEXT: v_or_b32_e32 v2, v0, v2
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
@@ -178,29 +190,29 @@ define void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i32> %x) {
; GFX7-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; GFX7-NEXT: v_bfe_u32 v6, v1, 8, 8
; GFX7-NEXT: ds_write_b8 v0, v1
-; GFX7-NEXT: ds_write_b8 v0, v6 offset:1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX7-NEXT: ds_write_b8 v0, v5 offset:2
; GFX7-NEXT: ds_write_b8 v0, v1 offset:3
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX7-NEXT: v_bfe_u32 v5, v2, 8, 8
; GFX7-NEXT: ds_write_b8 v0, v2 offset:4
-; GFX7-NEXT: ds_write_b8 v0, v5 offset:5
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
-; GFX7-NEXT: ds_write_b8 v0, v1 offset:6
; GFX7-NEXT: ds_write_b8 v0, v2 offset:7
; GFX7-NEXT: v_bfe_u32 v2, v3, 8, 8
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v3
-; GFX7-NEXT: ds_write_b8 v0, v3 offset:8
; GFX7-NEXT: ds_write_b8 v0, v2 offset:9
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v3
-; GFX7-NEXT: ds_write_b8 v0, v1 offset:10
+; GFX7-NEXT: ds_write_b8 v0, v1 offset:6
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX7-NEXT: ds_write_b8 v0, v2 offset:11
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX7-NEXT: v_bfe_u32 v2, v4, 8, 8
-; GFX7-NEXT: ds_write_b8 v0, v4 offset:12
+; GFX7-NEXT: ds_write_b8 v0, v1 offset:10
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX7-NEXT: ds_write_b8 v0, v2 offset:13
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v4
+; GFX7-NEXT: ds_write_b8 v0, v6 offset:1
+; GFX7-NEXT: ds_write_b8 v0, v5 offset:5
+; GFX7-NEXT: ds_write_b8 v0, v3 offset:8
+; GFX7-NEXT: ds_write_b8 v0, v4 offset:12
; GFX7-NEXT: ds_write_b8 v0, v1 offset:14
; GFX7-NEXT: ds_write_b8 v0, v2 offset:15
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
@@ -239,22 +251,22 @@ define void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i32> %x) {
; GFX7-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX7-NEXT: v_bfe_u32 v5, v1, 8, 8
; GFX7-NEXT: ds_write_b8 v0, v1
-; GFX7-NEXT: ds_write_b8 v0, v5 offset:1
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX7-NEXT: ds_write_b8 v0, v4 offset:2
; GFX7-NEXT: ds_write_b8 v0, v1 offset:3
; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v2
; GFX7-NEXT: v_bfe_u32 v4, v2, 8, 8
; GFX7-NEXT: ds_write_b8 v0, v2 offset:4
-; GFX7-NEXT: ds_write_b8 v0, v4 offset:5
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v2
-; GFX7-NEXT: ds_write_b8 v0, v1 offset:6
; GFX7-NEXT: ds_write_b8 v0, v2 offset:7
-; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX7-NEXT: v_bfe_u32 v2, v3, 8, 8
-; GFX7-NEXT: ds_write_b8 v0, v3 offset:8
+; GFX7-NEXT: ds_write_b8 v0, v1 offset:6
+; GFX7-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; GFX7-NEXT: ds_write_b8 v0, v2 offset:9
; GFX7-NEXT: v_lshrrev_b32_e32 v2, 24, v3
+; GFX7-NEXT: ds_write_b8 v0, v5 offset:1
+; GFX7-NEXT: ds_write_b8 v0, v4 offset:5
+; GFX7-NEXT: ds_write_b8 v0, v3 offset:8
; GFX7-NEXT: ds_write_b8 v0, v1 offset:10
; GFX7-NEXT: ds_write_b8 v0, v2 offset:11
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
index 10e83b70a57d4..57060711a5333 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.128.ll
@@ -180,50 +180,50 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10
; GFX10-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: s_lshr_b32 s4, s0, 16
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: v_mov_b32_e32 v1, s6
+; GFX10-NEXT: s_lshr_b32 s8, s2, 16
+; GFX10-NEXT: s_lshr_b32 s4, s0, 16
; GFX10-NEXT: s_and_b32 s5, 0xffff, s0
; GFX10-NEXT: s_lshr_b32 s0, s0, 24
; GFX10-NEXT: s_and_b32 s7, 0xffff, s1
-; GFX10-NEXT: v_mov_b32_e32 v2, s1
-; GFX10-NEXT: s_lshr_b32 s6, s1, 16
-; GFX10-NEXT: v_mov_b32_e32 v4, s4
-; GFX10-NEXT: s_lshr_b32 s8, s2, 16
-; GFX10-NEXT: s_lshr_b32 s5, s5, 8
+; GFX10-NEXT: ds_write_b8 v1, v0
+; GFX10-NEXT: v_mov_b32_e32 v0, s8
; GFX10-NEXT: v_mov_b32_e32 v5, s0
; GFX10-NEXT: s_lshr_b32 s0, s7, 8
-; GFX10-NEXT: v_mov_b32_e32 v6, s6
-; GFX10-NEXT: v_mov_b32_e32 v8, s5
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
; GFX10-NEXT: v_mov_b32_e32 v9, s0
-; GFX10-NEXT: ds_write_b8 v1, v0
-; GFX10-NEXT: ds_write_b8 v1, v2 offset:4
-; GFX10-NEXT: ds_write_b8 v1, v4 offset:2
-; GFX10-NEXT: ds_write_b8 v1, v5 offset:3
-; GFX10-NEXT: ds_write_b8 v1, v6 offset:6
-; GFX10-NEXT: ds_write_b8 v1, v8 offset:1
-; GFX10-NEXT: ds_write_b8 v1, v9 offset:5
-; GFX10-NEXT: v_mov_b32_e32 v0, s8
+; GFX10-NEXT: s_lshr_b32 s0, s2, 24
+; GFX10-NEXT: s_lshr_b32 s6, s1, 16
+; GFX10-NEXT: v_mov_b32_e32 v2, s1
; GFX10-NEXT: s_lshr_b32 s1, s1, 24
; GFX10-NEXT: s_and_b32 s9, 0xffff, s2
-; GFX10-NEXT: s_lshr_b32 s0, s2, 24
-; GFX10-NEXT: v_mov_b32_e32 v7, s1
-; GFX10-NEXT: s_lshr_b32 s1, s9, 8
-; GFX10-NEXT: v_mov_b32_e32 v3, s2
; GFX10-NEXT: ds_write_b8 v1, v0 offset:10
; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: s_and_b32 s0, 0xffff, s3
-; GFX10-NEXT: v_mov_b32_e32 v10, s1
+; GFX10-NEXT: v_mov_b32_e32 v4, s4
+; GFX10-NEXT: v_mov_b32_e32 v7, s1
+; GFX10-NEXT: s_lshr_b32 s1, s9, 8
; GFX10-NEXT: s_lshr_b32 s0, s0, 8
-; GFX10-NEXT: s_lshr_b32 s1, s3, 16
-; GFX10-NEXT: v_mov_b32_e32 v2, s3
-; GFX10-NEXT: ds_write_b8 v1, v7 offset:7
+; GFX10-NEXT: s_lshr_b32 s5, s5, 8
+; GFX10-NEXT: v_mov_b32_e32 v10, s1
; GFX10-NEXT: ds_write_b8 v1, v3 offset:8
-; GFX10-NEXT: ds_write_b8 v1, v10 offset:9
+; GFX10-NEXT: s_lshr_b32 s1, s3, 16
; GFX10-NEXT: v_mov_b32_e32 v3, s0
; GFX10-NEXT: s_lshr_b32 s0, s3, 24
+; GFX10-NEXT: v_mov_b32_e32 v6, s6
+; GFX10-NEXT: v_mov_b32_e32 v8, s5
+; GFX10-NEXT: ds_write_b8 v1, v2 offset:4
+; GFX10-NEXT: ds_write_b8 v1, v4 offset:2
+; GFX10-NEXT: ds_write_b8 v1, v5 offset:3
+; GFX10-NEXT: v_mov_b32_e32 v2, s3
; GFX10-NEXT: v_mov_b32_e32 v4, s1
; GFX10-NEXT: v_mov_b32_e32 v5, s0
+; GFX10-NEXT: ds_write_b8 v1, v6 offset:6
+; GFX10-NEXT: ds_write_b8 v1, v8 offset:1
+; GFX10-NEXT: ds_write_b8 v1, v9 offset:5
+; GFX10-NEXT: ds_write_b8 v1, v7 offset:7
+; GFX10-NEXT: ds_write_b8 v1, v10 offset:9
; GFX10-NEXT: ds_write_b8 v1, v0 offset:11
; GFX10-NEXT: ds_write_b8 v1, v2 offset:12
; GFX10-NEXT: ds_write_b8 v1, v3 offset:13
@@ -237,49 +237,50 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT: s_load_b32 s4, s[4:5], 0x0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_and_b32 s6, 0xffff, s0
-; GFX11-NEXT: s_lshr_b32 s5, s0, 16
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
+; GFX11-NEXT: s_lshr_b32 s8, s2, 16
+; GFX11-NEXT: s_lshr_b32 s5, s0, 16
+; GFX11-NEXT: s_and_b32 s6, 0xffff, s0
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
-; GFX11-NEXT: s_lshr_b32 s4, s1, 16
; GFX11-NEXT: s_and_b32 s7, 0xffff, s1
+; GFX11-NEXT: ds_store_b8 v1, v0
+; GFX11-NEXT: v_mov_b32_e32 v0, s8
; GFX11-NEXT: s_lshr_b32 s6, s6, 8
-; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
-; GFX11-NEXT: s_lshr_b32 s1, s1, 24
; GFX11-NEXT: v_dual_mov_b32 v4, s5 :: v_dual_mov_b32 v5, s0
; GFX11-NEXT: s_lshr_b32 s0, s7, 8
-; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s6
-; GFX11-NEXT: s_and_b32 s9, 0xffff, s2
-; GFX11-NEXT: v_dual_mov_b32 v8, s1 :: v_dual_mov_b32 v9, s0
+; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
+; GFX11-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v9, s0
; GFX11-NEXT: s_lshr_b32 s0, s2, 24
-; GFX11-NEXT: s_lshr_b32 s1, s9, 8
-; GFX11-NEXT: ds_store_b8 v1, v0
-; GFX11-NEXT: ds_store_b8 v1, v7 offset:1
-; GFX11-NEXT: ds_store_b8 v1, v4 offset:2
-; GFX11-NEXT: ds_store_b8 v1, v5 offset:3
-; GFX11-NEXT: ds_store_b8 v1, v2 offset:4
-; GFX11-NEXT: ds_store_b8 v1, v9 offset:5
-; GFX11-NEXT: ds_store_b8 v1, v6 offset:6
-; GFX11-NEXT: ds_store_b8 v1, v8 offset:7
-; GFX11-NEXT: v_mov_b32_e32 v4, s0
+; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT: s_lshr_b32 s4, s1, 16
+; GFX11-NEXT: s_lshr_b32 s1, s1, 24
+; GFX11-NEXT: s_and_b32 s9, 0xffff, s2
+; GFX11-NEXT: ds_store_b8 v1, v0 offset:10
+; GFX11-NEXT: v_mov_b32_e32 v0, s0
; GFX11-NEXT: s_and_b32 s0, 0xffff, s3
-; GFX11-NEXT: s_lshr_b32 s8, s2, 16
-; GFX11-NEXT: v_dual_mov_b32 v0, s1 :: v_dual_mov_b32 v5, s3
+; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s1
+; GFX11-NEXT: s_lshr_b32 s1, s9, 8
; GFX11-NEXT: s_lshr_b32 s0, s0, 8
+; GFX11-NEXT: v_mov_b32_e32 v10, s1
+; GFX11-NEXT: ds_store_b8 v1, v3 offset:8
; GFX11-NEXT: s_lshr_b32 s1, s3, 16
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v2, s8 :: v_dual_mov_b32 v7, s1
-; GFX11-NEXT: v_mov_b32_e32 v6, s0
+; GFX11-NEXT: v_mov_b32_e32 v3, s0
; GFX11-NEXT: s_lshr_b32 s0, s3, 24
-; GFX11-NEXT: v_mov_b32_e32 v8, s0
-; GFX11-NEXT: ds_store_b8 v1, v3 offset:8
-; GFX11-NEXT: ds_store_b8 v1, v0 offset:9
-; GFX11-NEXT: ds_store_b8 v1, v2 offset:10
-; GFX11-NEXT: ds_store_b8 v1, v4 offset:11
-; GFX11-NEXT: ds_store_b8 v1, v5 offset:12
-; GFX11-NEXT: ds_store_b8 v1, v6 offset:13
-; GFX11-NEXT: ds_store_b8 v1, v7 offset:14
-; GFX11-NEXT: ds_store_b8 v1, v8 offset:15
+; GFX11-NEXT: ds_store_b8 v1, v2 offset:4
+; GFX11-NEXT: ds_store_b8 v1, v4 offset:2
+; GFX11-NEXT: ds_store_b8 v1, v5 offset:3
+; GFX11-NEXT: v_mov_b32_e32 v2, s3
+; GFX11-NEXT: v_dual_mov_b32 v4, s1 :: v_dual_mov_b32 v5, s0
+; GFX11-NEXT: ds_store_b8 v1, v6 offset:6
+; GFX11-NEXT: ds_store_b8 v1, v8 offset:1
+; GFX11-NEXT: ds_store_b8 v1, v9 offset:5
+; GFX11-NEXT: ds_store_b8 v1, v7 offset:7
+; GFX11-NEXT: ds_store_b8 v1, v10 offset:9
+; GFX11-NEXT: ds_store_b8 v1, v0 offset:11
+; GFX11-NEXT: ds_store_b8 v1, v2 offset:12
+; GFX11-NEXT: ds_store_b8 v1, v3 offset:13
+; GFX11-NEXT: ds_store_b8 v1, v4 offset:14
+; GFX11-NEXT: ds_store_b8 v1, v5 offset:15
; GFX11-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out, align 1
ret void
@@ -349,14 +350,14 @@ define amdgpu_kernel void @store_lds_v4i32_align2(ptr addrspace(3) %out, <4 x i3
; GFX10-NEXT: s_load_dwordx4 s[0:3], s[4:5], 0x10
; GFX10-NEXT: s_load_dword s6, s[4:5], 0x0
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: v_mov_b32_e32 v1, s6
-; GFX10-NEXT: v_mov_b32_e32 v2, s1
; GFX10-NEXT: s_lshr_b32 s4, s0, 16
-; GFX10-NEXT: v_mov_b32_e32 v3, s2
+; GFX10-NEXT: v_mov_b32_e32 v0, s0
; GFX10-NEXT: s_lshr_b32 s0, s1, 16
+; GFX10-NEXT: v_mov_b32_e32 v2, s1
; GFX10-NEXT: s_lshr_b32 s1, s2, 16
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
; GFX10-NEXT: s_lshr_b32 s2, s3, 16
+; GFX10-NEXT: v_mov_b32_e32 v1, s6
; GFX10-NEXT: v_mov_b32_e32 v4, s3
; GFX10-NEXT: v_mov_b32_e32 v5, s4
; GFX10-NEXT: v_mov_b32_e32 v6, s0
@@ -381,19 +382,19 @@ define amdgpu_kernel void @store_lds_v4i32_align2(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: s_lshr_b32 s5, s0, 16
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s4
; GFX11-NEXT: s_lshr_b32 s0, s1, 16
-; GFX11-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s5
; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
; GFX11-NEXT: s_lshr_b32 s1, s2, 16
; GFX11-NEXT: s_lshr_b32 s2, s3, 16
+; GFX11-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s5
; GFX11-NEXT: v_dual_mov_b32 v6, s0 :: v_dual_mov_b32 v7, s1
; GFX11-NEXT: v_mov_b32_e32 v8, s2
; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: ds_store_b16 v1, v5 offset:2
; GFX11-NEXT: ds_store_b16 v1, v2 offset:4
-; GFX11-NEXT: ds_store_b16 v1, v6 offset:6
; GFX11-NEXT: ds_store_b16 v1, v3 offset:8
-; GFX11-NEXT: ds_store_b16 v1, v7 offset:10
; GFX11-NEXT: ds_store_b16 v1, v4 offset:12
+; GFX11-NEXT: ds_store_b16 v1, v5 offset:2
+; GFX11-NEXT: ds_store_b16 v1, v6 offset:6
+; GFX11-NEXT: ds_store_b16 v1, v7 offset:10
; GFX11-NEXT: ds_store_b16 v1, v8 offset:14
; GFX11-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out, align 2
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
index db5d0fdb5c036..27a22e8b3a7f4 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/store-local.96.ll
@@ -163,36 +163,36 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX10-NEXT: s_lshr_b32 s4, s0, 16
; GFX10-NEXT: s_and_b32 s5, 0xffff, s0
; GFX10-NEXT: v_mov_b32_e32 v0, s0
-; GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: s_lshr_b32 s0, s0, 24
-; GFX10-NEXT: s_lshr_b32 s3, s1, 16
; GFX10-NEXT: s_and_b32 s6, 0xffff, s1
+; GFX10-NEXT: s_waitcnt lgkmcnt(0)
+; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: v_mov_b32_e32 v2, s1
+; GFX10-NEXT: s_lshr_b32 s3, s1, 16
; GFX10-NEXT: s_lshr_b32 s1, s1, 24
; GFX10-NEXT: s_and_b32 s8, 0xffff, s2
-; GFX10-NEXT: v_mov_b32_e32 v4, s4
; GFX10-NEXT: v_mov_b32_e32 v5, s0
; GFX10-NEXT: s_lshr_b32 s0, s6, 8
-; GFX10-NEXT: v_mov_b32_e32 v7, s1
-; GFX10-NEXT: s_lshr_b32 s1, s8, 8
; GFX10-NEXT: s_lshr_b32 s7, s2, 16
-; GFX10-NEXT: v_mov_b32_e32 v3, s2
; GFX10-NEXT: s_lshr_b32 s5, s5, 8
-; GFX10-NEXT: v_mov_b32_e32 v6, s3
+; GFX10-NEXT: v_mov_b32_e32 v7, s1
+; GFX10-NEXT: s_lshr_b32 s1, s8, 8
; GFX10-NEXT: v_mov_b32_e32 v9, s0
-; GFX10-NEXT: v_mov_b32_e32 v10, s1
; GFX10-NEXT: s_lshr_b32 s0, s2, 24
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
+; GFX10-NEXT: v_mov_b32_e32 v4, s4
+; GFX10-NEXT: v_mov_b32_e32 v6, s3
; GFX10-NEXT: v_mov_b32_e32 v8, s5
+; GFX10-NEXT: v_mov_b32_e32 v10, s1
; GFX10-NEXT: ds_write_b8 v1, v0
; GFX10-NEXT: ds_write_b8 v1, v2 offset:4
+; GFX10-NEXT: v_mov_b32_e32 v0, s7
+; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: ds_write_b8 v1, v4 offset:2
; GFX10-NEXT: ds_write_b8 v1, v5 offset:3
; GFX10-NEXT: ds_write_b8 v1, v6 offset:6
; GFX10-NEXT: ds_write_b8 v1, v8 offset:1
; GFX10-NEXT: ds_write_b8 v1, v9 offset:5
-; GFX10-NEXT: v_mov_b32_e32 v0, s7
-; GFX10-NEXT: v_mov_b32_e32 v2, s0
; GFX10-NEXT: ds_write_b8 v1, v7 offset:7
; GFX10-NEXT: ds_write_b8 v1, v3 offset:8
; GFX10-NEXT: ds_write_b8 v1, v10 offset:9
@@ -205,38 +205,39 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_load_b32 s3, s[4:5], 0x0
-; GFX11-NEXT: s_and_b32 s5, 0xffff, s0
-; GFX11-NEXT: s_lshr_b32 s7, s2, 16
-; GFX11-NEXT: s_lshr_b32 s5, s5, 8
; GFX11-NEXT: s_lshr_b32 s4, s0, 16
+; GFX11-NEXT: s_and_b32 s5, 0xffff, s0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
-; GFX11-NEXT: s_lshr_b32 s3, s1, 16
; GFX11-NEXT: s_and_b32 s6, 0xffff, s1
; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
+; GFX11-NEXT: s_lshr_b32 s3, s1, 16
; GFX11-NEXT: s_lshr_b32 s1, s1, 24
; GFX11-NEXT: s_and_b32 s8, 0xffff, s2
-; GFX11-NEXT: v_dual_mov_b32 v8, s7 :: v_dual_mov_b32 v9, s5
+; GFX11-NEXT: s_lshr_b32 s5, s5, 8
; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s0
-; GFX11-NEXT: s_lshr_b32 s2, s2, 24
; GFX11-NEXT: s_lshr_b32 s0, s6, 8
+; GFX11-NEXT: s_lshr_b32 s7, s2, 16
; GFX11-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s1
; GFX11-NEXT: s_lshr_b32 s1, s8, 8
-; GFX11-NEXT: v_dual_mov_b32 v10, s2 :: v_dual_mov_b32 v11, s0
-; GFX11-NEXT: v_mov_b32_e32 v12, s1
+; GFX11-NEXT: v_dual_mov_b32 v8, s5 :: v_dual_mov_b32 v9, s0
+; GFX11-NEXT: s_lshr_b32 s0, s2, 24
+; GFX11-NEXT: v_mov_b32_e32 v10, s1
; GFX11-NEXT: ds_store_b8 v1, v0
-; GFX11-NEXT: ds_store_b8 v1, v9 offset:1
+; GFX11-NEXT: ds_store_b8 v1, v2 offset:4
+; GFX11-NEXT: v_mov_b32_e32 v0, s7
+; GFX11-NEXT: v_mov_b32_e32 v2, s0
; GFX11-NEXT: ds_store_b8 v1, v4 offset:2
; GFX11-NEXT: ds_store_b8 v1, v5 offset:3
-; GFX11-NEXT: ds_store_b8 v1, v2 offset:4
-; GFX11-NEXT: ds_store_b8 v1, v11 offset:5
; GFX11-NEXT: ds_store_b8 v1, v6 offset:6
+; GFX11-NEXT: ds_store_b8 v1, v8 offset:1
+; GFX11-NEXT: ds_store_b8 v1, v9 offset:5
; GFX11-NEXT: ds_store_b8 v1, v7 offset:7
; GFX11-NEXT: ds_store_b8 v1, v3 offset:8
-; GFX11-NEXT: ds_store_b8 v1, v12 offset:9
-; GFX11-NEXT: ds_store_b8 v1, v8 offset:10
-; GFX11-NEXT: ds_store_b8 v1, v10 offset:11
+; GFX11-NEXT: ds_store_b8 v1, v10 offset:9
+; GFX11-NEXT: ds_store_b8 v1, v0 offset:10
+; GFX11-NEXT: ds_store_b8 v1, v2 offset:11
; GFX11-NEXT: s_endpgm
store <3 x i32> %x, ptr addrspace(3) %out, align 1
ret void
@@ -299,14 +300,14 @@ define amdgpu_kernel void @store_lds_v3i32_align2(ptr addrspace(3) %out, <3 x i3
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_load_dword s3, s[4:5], 0x0
; GFX10-NEXT: ; kill: killed $sgpr4_sgpr5
+; GFX10-NEXT: s_lshr_b32 s4, s0, 16
; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: s_lshr_b32 s0, s1, 16
+; GFX10-NEXT: v_mov_b32_e32 v2, s1
+; GFX10-NEXT: s_lshr_b32 s1, s2, 16
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: v_mov_b32_e32 v1, s3
-; GFX10-NEXT: s_lshr_b32 s4, s0, 16
-; GFX10-NEXT: v_mov_b32_e32 v2, s1
-; GFX10-NEXT: s_lshr_b32 s0, s1, 16
; GFX10-NEXT: v_mov_b32_e32 v3, s2
-; GFX10-NEXT: s_lshr_b32 s1, s2, 16
; GFX10-NEXT: v_mov_b32_e32 v4, s4
; GFX10-NEXT: v_mov_b32_e32 v5, s0
; GFX10-NEXT: v_mov_b32_e32 v6, s1
@@ -327,15 +328,15 @@ define amdgpu_kernel void @store_lds_v3i32_align2(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: s_lshr_b32 s0, s1, 16
-; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s4
+; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
; GFX11-NEXT: s_lshr_b32 s1, s2, 16
-; GFX11-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v5, s0
+; GFX11-NEXT: v_dual_mov_b32 v4, s4 :: v_dual_mov_b32 v5, s0
; GFX11-NEXT: v_mov_b32_e32 v6, s1
; GFX11-NEXT: ds_store_b16 v1, v0
-; GFX11-NEXT: ds_store_b16 v1, v3 offset:2
; GFX11-NEXT: ds_store_b16 v1, v2 offset:4
+; GFX11-NEXT: ds_store_b16 v1, v3 offset:8
+; GFX11-NEXT: ds_store_b16 v1, v4 offset:2
; GFX11-NEXT: ds_store_b16 v1, v5 offset:6
-; GFX11-NEXT: ds_store_b16 v1, v4 offset:8
; GFX11-NEXT: ds_store_b16 v1, v6 offset:10
; GFX11-NEXT: s_endpgm
store <3 x i32> %x, ptr addrspace(3) %out, align 2
diff --git a/llvm/test/CodeGen/AMDGPU/atomic_store_local.ll b/llvm/test/CodeGen/AMDGPU/atomic_store_local.ll
index 31065f2b225a9..dfe6f8f21cda7 100644
--- a/llvm/test/CodeGen/AMDGPU/atomic_store_local.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomic_store_local.ll
@@ -10,8 +10,8 @@ define void @atomic_store_monotonic_i8(ptr addrspace(3) %ptr, i8 %val) {
; CI-LABEL: atomic_store_monotonic_i8:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1
+; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: ds_write_b8 v0, v1
; CI-NEXT: ds_write_b8 v0, v2
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -73,8 +73,8 @@ define void @atomic_store_monotonic_offset_i8(ptr addrspace(3) %ptr, i8 %val) {
; CI-LABEL: atomic_store_monotonic_offset_i8:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1
+; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: ds_write_b8 v0, v1 offset:8
; CI-NEXT: ds_write_b8 v0, v2 offset:16
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -138,8 +138,8 @@ define void @atomic_store_monotonic_i16(ptr addrspace(3) %ptr, i16 %val) {
; CI-LABEL: atomic_store_monotonic_i16:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1
+; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: ds_write_b16 v0, v1
; CI-NEXT: ds_write_b16 v0, v2
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -201,8 +201,8 @@ define void @atomic_store_monotonic_offset_i16(ptr addrspace(3) %ptr, i16 %val)
; CI-LABEL: atomic_store_monotonic_offset_i16:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1
+; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: ds_write_b16 v0, v1 offset:32
; CI-NEXT: ds_write_b16 v0, v2 offset:32
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -405,8 +405,8 @@ define void @atomic_store_monotonic_f16(ptr addrspace(3) %ptr, i16 %arg.val) {
; CI-LABEL: atomic_store_monotonic_f16:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1
+; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: ds_write_b16 v0, v1
; CI-NEXT: ds_write_b16 v0, v2
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -470,8 +470,8 @@ define void @atomic_store_monotonic_offset_f16(ptr addrspace(3) %ptr, i16 %arg.v
; CI-LABEL: atomic_store_monotonic_offset_f16:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1
+; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: ds_write_b16 v0, v1 offset:32
; CI-NEXT: ds_write_b16 v0, v2 offset:32
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -536,8 +536,8 @@ define void @atomic_store_monotonic_bf16(ptr addrspace(3) %ptr, i16 %arg.val) {
; CI-LABEL: atomic_store_monotonic_bf16:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1
+; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: ds_write_b16 v0, v1
; CI-NEXT: ds_write_b16 v0, v2
; CI-NEXT: s_waitcnt lgkmcnt(0)
@@ -601,8 +601,8 @@ define void @atomic_store_monotonic_offset_bf16(ptr addrspace(3) %ptr, i16 %arg.
; CI-LABEL: atomic_store_monotonic_offset_bf16:
; CI: ; %bb.0:
; CI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: v_add_i32_e32 v2, vcc, 2, v1
+; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: ds_write_b16 v0, v1 offset:32
; CI-NEXT: ds_write_b16 v0, v2 offset:32
; CI-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/ds-alignment.ll b/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
index 5dcf5d437bae6..eecf8803b83eb 100644
--- a/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds-alignment.ll
@@ -271,29 +271,30 @@ define amdgpu_kernel void @ds8align1(ptr addrspace(3) %in, ptr addrspace(3) %out
; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:2
; ALIGNED-GISEL-NEXT: ds_read_u8 v4, v0 offset:3
; ALIGNED-GISEL-NEXT: ds_read_u8 v5, v0 offset:4
-; ALIGNED-GISEL-NEXT: ds_read_u8 v6, v0 offset:5
-; ALIGNED-GISEL-NEXT: ds_read_u8 v7, v0 offset:6
-; ALIGNED-GISEL-NEXT: ds_read_u8 v0, v0 offset:7
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(6)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1
-; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 8
-; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0
+; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0 offset:5
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:6
+; ALIGNED-GISEL-NEXT: ds_read_u8 v0, v0 offset:7
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(5)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v3
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v3
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v4
-; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 24
-; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 16
-; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v4
+; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 8
+; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 24
+; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0
+; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s3, 16
+; ALIGNED-GISEL-NEXT: s_or_b32 s2, s4, s2
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v6
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v1
; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v5
; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 8
; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v7
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v2
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v0
; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 24
@@ -368,14 +369,14 @@ define amdgpu_kernel void @ds8align2(ptr addrspace(3) %in, ptr addrspace(3) %out
; ALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; ALIGNED-GISEL-NEXT: ds_read_u16 v1, v0
; ALIGNED-GISEL-NEXT: ds_read_u16 v2, v0 offset:2
+; ALIGNED-GISEL-NEXT: ds_read_u16 v1, v0
; ALIGNED-GISEL-NEXT: ds_read_u16 v3, v0 offset:4
; ALIGNED-GISEL-NEXT: ds_read_u16 v0, v0 offset:6
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(3)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1
; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 16
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v0
@@ -490,46 +491,48 @@ define amdgpu_kernel void @ds12align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:2
; ALIGNED-GISEL-NEXT: ds_read_u8 v4, v0 offset:3
; ALIGNED-GISEL-NEXT: ds_read_u8 v5, v0 offset:4
-; ALIGNED-GISEL-NEXT: ds_read_u8 v6, v0 offset:5
-; ALIGNED-GISEL-NEXT: ds_read_u8 v7, v0 offset:6
-; ALIGNED-GISEL-NEXT: ds_read_u8 v8, v0 offset:7
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(6)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0 offset:5
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(3)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:6
+; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:7
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v4
; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 8
+; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 24
; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(5)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v4
-; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 24
-; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 16
+; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s3, 16
+; ALIGNED-GISEL-NEXT: s_or_b32 s2, s4, s2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v1
; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0 offset:8
-; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:9
-; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v6
; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v5
; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 8
-; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:10
-; ALIGNED-GISEL-NEXT: ds_read_u8 v0, v0 offset:11
; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(5)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v7
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v8
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v3
; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 24
; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 16
+; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:9
; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v2
; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0 offset:10
+; ALIGNED-GISEL-NEXT: ds_read_u8 v0, v0 offset:11
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v2
; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 8
; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v3
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v1
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v0
; ALIGNED-GISEL-NEXT: s_lshl_b32 s5, s5, 24
@@ -621,18 +624,20 @@ define amdgpu_kernel void @ds12align2(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-GISEL-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
-; ALIGNED-GISEL-NEXT: ds_read_u16 v1, v0
; ALIGNED-GISEL-NEXT: ds_read_u16 v2, v0 offset:2
-; ALIGNED-GISEL-NEXT: ds_read_u16 v3, v0 offset:4
+; ALIGNED-GISEL-NEXT: ds_read_u16 v1, v0
; ALIGNED-GISEL-NEXT: ds_read_u16 v4, v0 offset:6
+; ALIGNED-GISEL-NEXT: ds_read_u16 v3, v0 offset:4
; ALIGNED-GISEL-NEXT: ds_read_u16 v5, v0 offset:8
; ALIGNED-GISEL-NEXT: ds_read_u16 v0, v0 offset:10
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(5)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(3)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v4
; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 16
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v3
; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0
; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s4, 16
@@ -847,63 +852,66 @@ define amdgpu_kernel void @ds16align1(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:2
; ALIGNED-GISEL-NEXT: ds_read_u8 v4, v0 offset:3
; ALIGNED-GISEL-NEXT: ds_read_u8 v5, v0 offset:4
-; ALIGNED-GISEL-NEXT: ds_read_u8 v6, v0 offset:5
-; ALIGNED-GISEL-NEXT: ds_read_u8 v7, v0 offset:6
-; ALIGNED-GISEL-NEXT: ds_read_u8 v8, v0 offset:7
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(6)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1
+; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0 offset:5
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(3)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:6
+; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:7
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v4
; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 8
+; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 24
; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(5)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v4
-; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 24
-; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 16
-; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2
+; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s3, 16
+; ALIGNED-GISEL-NEXT: s_or_b32 s2, s4, s2
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v6
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0 offset:8
; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v5
; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 8
; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v2
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v7
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v8
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v3
; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 24
; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 16
-; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3
-; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0 offset:8
; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:9
-; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:10
-; ALIGNED-GISEL-NEXT: ds_read_u8 v4, v0 offset:11
-; ALIGNED-GISEL-NEXT: ds_read_u8 v5, v0 offset:12
-; ALIGNED-GISEL-NEXT: ds_read_u8 v6, v0 offset:13
-; ALIGNED-GISEL-NEXT: ds_read_u8 v7, v0 offset:14
-; ALIGNED-GISEL-NEXT: ds_read_u8 v0, v0 offset:15
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(6)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v2
+; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3
; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v1
+; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0 offset:10
+; ALIGNED-GISEL-NEXT: ds_read_u8 v3, v0 offset:11
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v2
; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 8
; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(5)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v3
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v4
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0 offset:12
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v3
; ALIGNED-GISEL-NEXT: s_lshl_b32 s5, s5, 24
; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 16
+; ALIGNED-GISEL-NEXT: ds_read_u8 v2, v0 offset:13
; ALIGNED-GISEL-NEXT: s_or_b32 s4, s5, s4
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v6
; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v5
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v1
+; ALIGNED-GISEL-NEXT: ds_read_u8 v1, v0 offset:14
+; ALIGNED-GISEL-NEXT: ds_read_u8 v0, v0 offset:15
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v2
; ALIGNED-GISEL-NEXT: s_lshl_b32 s5, s5, 8
; ALIGNED-GISEL-NEXT: s_or_b32 s4, s5, s4
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(1)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v7
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v1
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s6, v0
; ALIGNED-GISEL-NEXT: s_lshl_b32 s6, s6, 24
@@ -1012,30 +1020,32 @@ define amdgpu_kernel void @ds16align2(ptr addrspace(3) %in, ptr addrspace(3) %ou
; ALIGNED-GISEL-NEXT: v_mov_b32_e32 v0, s0
; ALIGNED-GISEL-NEXT: ds_read_u16 v1, v0
; ALIGNED-GISEL-NEXT: ds_read_u16 v2, v0 offset:2
-; ALIGNED-GISEL-NEXT: ds_read_u16 v3, v0 offset:4
; ALIGNED-GISEL-NEXT: ds_read_u16 v4, v0 offset:6
+; ALIGNED-GISEL-NEXT: ds_read_u16 v3, v0 offset:4
; ALIGNED-GISEL-NEXT: ds_read_u16 v5, v0 offset:8
-; ALIGNED-GISEL-NEXT: ds_read_u16 v6, v0 offset:10
-; ALIGNED-GISEL-NEXT: ds_read_u16 v7, v0 offset:12
-; ALIGNED-GISEL-NEXT: ds_read_u16 v0, v0 offset:14
-; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(6)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s0, v1
-; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 16
+; ALIGNED-GISEL-NEXT: ds_read_u16 v1, v0 offset:10
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v2
+; ALIGNED-GISEL-NEXT: ds_read_u16 v2, v0 offset:12
+; ALIGNED-GISEL-NEXT: ds_read_u16 v0, v0 offset:14
+; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(5)
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v4
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(4)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v4
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v3
+; ALIGNED-GISEL-NEXT: s_lshl_b32 s2, s2, 16
+; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 16
; ALIGNED-GISEL-NEXT: s_or_b32 s0, s2, s0
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s2, v3
-; ALIGNED-GISEL-NEXT: s_lshl_b32 s3, s3, 16
+; ALIGNED-GISEL-NEXT: s_or_b32 s2, s4, s3
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(2)
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v6
-; ALIGNED-GISEL-NEXT: s_or_b32 s2, s3, s2
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v1
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s3, v5
; ALIGNED-GISEL-NEXT: s_lshl_b32 s4, s4, 16
; ALIGNED-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s5, v0
; ALIGNED-GISEL-NEXT: s_or_b32 s3, s4, s3
-; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v7
+; ALIGNED-GISEL-NEXT: v_readfirstlane_b32 s4, v2
; ALIGNED-GISEL-NEXT: s_lshl_b32 s5, s5, 16
; ALIGNED-GISEL-NEXT: s_or_b32 s4, s5, s4
; ALIGNED-GISEL-NEXT: s_lshr_b32 s5, s0, 16
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll b/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll
index 8116078c9ee71..f89cd7ac0a5e6 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_read2-gfx1250.ll
@@ -12,7 +12,7 @@ define amdgpu_kernel void @simple_read2_f32(ptr addrspace(1) %out) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v2, 0xffc, v0
; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:8
@@ -38,7 +38,7 @@ define amdgpu_kernel void @simple_read2_f32_max_offset(ptr addrspace(1) %out) #0
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v2, 0xffc, v0
; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:255
@@ -64,7 +64,7 @@ define amdgpu_kernel void @simple_read2_f32_too_far(ptr addrspace(1) %out) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
; GFX1250-NEXT: ds_load_b32 v1, v0
@@ -91,7 +91,7 @@ define amdgpu_kernel void @simple_read2_f32_x2(ptr addrspace(1) %out) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v4, 0xffc, v0
; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v4 offset1:8
@@ -133,7 +133,7 @@ define amdgpu_kernel void @simple_read2_f32_x2_barrier(ptr addrspace(1) %out) #0
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v4, 0xffc, v0
; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v4 offset1:8
@@ -181,7 +181,7 @@ define amdgpu_kernel void @simple_read2_f32_x2_nonzero_base(ptr addrspace(1) %ou
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v4, 0xffc, v0
; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v4 offset0:2 offset1:8
@@ -226,7 +226,7 @@ define amdgpu_kernel void @read2_ptr_is_subreg_arg_f32(ptr addrspace(1) %out, <2
; GFX1250-LABEL: read2_ptr_is_subreg_arg_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
@@ -259,7 +259,7 @@ define amdgpu_kernel void @read2_ptr_is_subreg_arg_offset_f32(ptr addrspace(1) %
; GFX1250-LABEL: read2_ptr_is_subreg_arg_offset_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, s3
@@ -293,7 +293,7 @@ define amdgpu_kernel void @read2_ptr_is_subreg_f32(ptr addrspace(1) %out) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v2, 0xffc, v0
; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:8
@@ -325,7 +325,7 @@ define amdgpu_kernel void @simple_read2_f32_volatile_0(ptr addrspace(1) %out) #0
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
; GFX1250-NEXT: ds_load_b32 v1, v0
@@ -352,7 +352,7 @@ define amdgpu_kernel void @simple_read2_f32_volatile_1(ptr addrspace(1) %out) #0
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
; GFX1250-NEXT: ds_load_b32 v1, v0
@@ -379,7 +379,7 @@ define amdgpu_kernel void @unaligned_read2_f32(ptr addrspace(1) %out, ptr addrsp
; GFX1250-LABEL: unaligned_read2_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v2, 0xffc, v0
@@ -406,7 +406,7 @@ define amdgpu_kernel void @unaligned_offset_read2_f32(ptr addrspace(1) %out, ptr
; GFX1250-LABEL: unaligned_offset_read2_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v2, 0xffc, v0
@@ -433,7 +433,7 @@ define amdgpu_kernel void @misaligned_2_simple_read2_f32(ptr addrspace(1) %out,
; GFX1250-LABEL: misaligned_2_simple_read2_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v2, 0xffc, v0
@@ -461,7 +461,7 @@ define amdgpu_kernel void @simple_read2_f64(ptr addrspace(1) %out) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v4, 0x1ff8, v0
; GFX1250-NEXT: ds_load_2addr_b64 v[0:3], v4 offset1:8
@@ -488,7 +488,7 @@ define amdgpu_kernel void @simple_read2_f64_max_offset(ptr addrspace(1) %out) #0
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v4, 0x1ff8, v0
; GFX1250-NEXT: ds_load_2addr_b64 v[0:3], v4 offset1:255
@@ -515,7 +515,7 @@ define amdgpu_kernel void @simple_read2_f64_too_far(ptr addrspace(1) %out) #0 {
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v4, 0x1ff8, v0
; GFX1250-NEXT: ds_load_b64 v[0:1], v4
@@ -543,7 +543,7 @@ define amdgpu_kernel void @misaligned_read2_f64(ptr addrspace(1) %out, ptr addrs
; GFX1250-LABEL: misaligned_read2_f64:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v4, 0x1ff8, v0
@@ -575,7 +575,7 @@ define amdgpu_kernel void @load_constant_adjacent_offsets(ptr addrspace(1) %out)
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: ds_load_b64 v[0:1], v2
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: v_add_nc_u32_e32 v0, v0, v1
@@ -594,7 +594,7 @@ define amdgpu_kernel void @load_constant_disjoint_offsets(ptr addrspace(1) %out)
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:2
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: v_add_nc_u32_e32 v0, v0, v1
@@ -615,7 +615,7 @@ define amdgpu_kernel void @load_misaligned64_constant_offsets(ptr addrspace(1) %
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: ds_load_b128 v[0:3], v4
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: v_add_nc_u64_e32 v[0:1], v[0:1], v[2:3]
@@ -636,7 +636,7 @@ define amdgpu_kernel void @load_misaligned64_constant_large_offsets(ptr addrspac
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_mov_b32_e32 v4, 0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: ds_load_b64 v[0:1], v4 offset:16384
; GFX1250-NEXT: ds_load_b64 v[2:3], v4 offset:32760
; GFX1250-NEXT: s_wait_dscnt 0x0
@@ -672,31 +672,31 @@ define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(ptr addrspace(1) %C,
; GFX1250-NEXT: s_add_co_i32 s1, s0, 0xc20
; GFX1250-NEXT: s_addk_co_i32 s0, 0xc60
; GFX1250-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v4, s0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: ds_load_2addr_b32 v[2:3], v1 offset1:1
; GFX1250-NEXT: ds_load_2addr_b32 v[4:5], v4 offset1:1
; GFX1250-NEXT: s_wait_dscnt 0x1
-; GFX1250-NEXT: v_dual_lshrrev_b32 v0, 8, v0 :: v_dual_add_f32 v2, v2, v3
+; GFX1250-NEXT: v_dual_add_f32 v7, v2, v3 :: v_dual_lshrrev_b32 v0, 8, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_and_b32_e32 v8, 0xffc, v0
-; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v8 offset1:1
-; GFX1250-NEXT: ds_load_2addr_b32 v[6:7], v8 offset0:32 offset1:33
+; GFX1250-NEXT: v_and_b32_e32 v6, 0xffc, v0
+; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v6 offset1:1
+; GFX1250-NEXT: ds_load_2addr_b32 v[2:3], v6 offset0:32 offset1:33
; GFX1250-NEXT: s_wait_dscnt 0x2
-; GFX1250-NEXT: v_add_f32_e32 v2, v2, v4
-; GFX1250-NEXT: v_add_f32_e32 v4, v2, v5
-; GFX1250-NEXT: ds_load_2addr_b32 v[2:3], v8 offset0:64 offset1:65
+; GFX1250-NEXT: v_add_f32_e32 v4, v7, v4
+; GFX1250-NEXT: v_add_f32_e32 v7, v4, v5
+; GFX1250-NEXT: ds_load_2addr_b32 v[4:5], v6 offset0:64 offset1:65
; GFX1250-NEXT: s_wait_dscnt 0x2
-; GFX1250-NEXT: v_add_f32_e32 v0, v4, v0
+; GFX1250-NEXT: v_add_f32_e32 v0, v7, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX1250-NEXT: v_dual_add_f32 v0, v0, v1 :: v_dual_mov_b32 v1, 0
; GFX1250-NEXT: s_wait_dscnt 0x1
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v6
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v2
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v7
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v3
; GFX1250-NEXT: s_wait_dscnt 0x0
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v2
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v4
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX1250-NEXT: v_add_f32_e32 v0, v0, v5
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX1250-NEXT: s_endpgm
@@ -747,7 +747,7 @@ define amdgpu_kernel void @misaligned_read2_v2i32(ptr addrspace(1) %out, ptr add
; GFX1250-LABEL: misaligned_read2_v2i32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v0 offset1:1
@@ -763,7 +763,7 @@ define amdgpu_kernel void @misaligned_read2_i64(ptr addrspace(1) %out, ptr addrs
; GFX1250-LABEL: misaligned_read2_i64:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x0 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_mov_b32 v0, s2
; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v0 offset1:1
@@ -779,7 +779,7 @@ define amdgpu_kernel void @ds_read_diff_base_interleaving(
; GFX1250-LABEL: ds_read_diff_base_interleaving:
; GFX1250: ; %bb.0: ; %bb
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_dual_lshrrev_b32 v1, 6, v0 :: v_dual_lshlrev_b32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-NEXT: v_and_b32_e32 v1, 0x3ff0, v1
@@ -788,7 +788,7 @@ define amdgpu_kernel void @ds_read_diff_base_interleaving(
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_dual_add_nc_u32 v2, s0, v1 :: v_dual_add_nc_u32 v3, s1, v0
; GFX1250-NEXT: v_dual_add_nc_u32 v4, s2, v1 :: v_dual_add_nc_u32 v6, s3, v0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:1
; GFX1250-NEXT: ds_load_2addr_b32 v[2:3], v3 offset1:4
; GFX1250-NEXT: ds_load_2addr_b32 v[4:5], v4 offset1:1
@@ -847,7 +847,7 @@ define amdgpu_kernel void @ds_read_call_read(ptr addrspace(1) %out, ptr addrspac
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_mov_b32 s32, 0
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[36:38], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b96 s[36:38], s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v1, 0x3ff, v0
; GFX1250-NEXT: v_dual_mov_b32 v42, 0 :: v_dual_mov_b32 v31, v0
; GFX1250-NEXT: s_mov_b64 s[10:11], s[6:7]
@@ -883,7 +883,7 @@ define amdgpu_kernel void @read2_v2i32_align1_odd_offset(ptr addrspace(1) %out)
; GFX1250: ; %bb.0: ; %entry
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: v_mov_b32_e32 v2, 0
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x0 nv
; GFX1250-NEXT: ds_load_b64 v[0:1], v2 offset:65
; GFX1250-NEXT: s_wait_dscnt 0x0
; GFX1250-NEXT: s_wait_kmcnt 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/ds_read2.ll b/llvm/test/CodeGen/AMDGPU/ds_read2.ll
index 9f1b55ea3b1ef..09e2a02120e96 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_read2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_read2.ll
@@ -522,39 +522,40 @@ define amdgpu_kernel void @simple_read2_f32_volatile_1(ptr addrspace(1) %out) #0
define amdgpu_kernel void @unaligned_read2_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
; CI-LABEL: unaligned_read2_f32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CI-NEXT: s_load_dword s0, s[4:5], 0x2
; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_add_i32_e32 v1, vcc, s2, v0
+; CI-NEXT: v_add_i32_e32 v1, vcc, s0, v0
; CI-NEXT: ds_read_u8 v2, v1 offset:1
-; CI-NEXT: ds_read_u8 v3, v1 offset:34
-; CI-NEXT: ds_read_u8 v4, v1 offset:32
+; CI-NEXT: ds_read_u8 v3, v1
+; CI-NEXT: ds_read_u8 v4, v1 offset:3
; CI-NEXT: ds_read_u8 v5, v1 offset:2
-; CI-NEXT: ds_read_u8 v6, v1
-; CI-NEXT: ds_read_u8 v7, v1 offset:3
-; CI-NEXT: ds_read_u8 v8, v1 offset:33
-; CI-NEXT: ds_read_u8 v1, v1 offset:35
-; CI-NEXT: s_waitcnt lgkmcnt(7)
+; CI-NEXT: ds_read_u8 v6, v1 offset:35
+; CI-NEXT: s_waitcnt lgkmcnt(4)
; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
; CI-NEXT: s_waitcnt lgkmcnt(3)
-; CI-NEXT: v_or_b32_e32 v2, v2, v6
+; CI-NEXT: v_or_b32_e32 v2, v2, v3
+; CI-NEXT: s_waitcnt lgkmcnt(2)
+; CI-NEXT: v_lshlrev_b32_e32 v3, 8, v4
+; CI-NEXT: ds_read_u8 v4, v1 offset:33
; CI-NEXT: s_waitcnt lgkmcnt(2)
-; CI-NEXT: v_lshlrev_b32_e32 v6, 8, v7
-; CI-NEXT: v_or_b32_e32 v5, v6, v5
-; CI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; CI-NEXT: v_or_b32_e32 v3, v3, v5
+; CI-NEXT: ds_read_u8 v5, v1 offset:32
+; CI-NEXT: ds_read_u8 v1, v1 offset:34
+; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CI-NEXT: v_or_b32_e32 v2, v3, v2
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; CI-NEXT: v_or_b32_e32 v2, v5, v2
-; CI-NEXT: v_lshlrev_b32_e32 v5, 8, v8
-; CI-NEXT: v_or_b32_e32 v1, v1, v3
-; CI-NEXT: v_or_b32_e32 v4, v5, v4
+; CI-NEXT: v_lshlrev_b32_e32 v3, 8, v4
+; CI-NEXT: v_lshlrev_b32_e32 v4, 8, v6
+; CI-NEXT: v_or_b32_e32 v1, v4, v1
+; CI-NEXT: v_or_b32_e32 v3, v3, v5
; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_or_b32_e32 v1, v1, v4
+; CI-NEXT: v_or_b32_e32 v1, v1, v3
; CI-NEXT: v_add_f32_e32 v2, v2, v1
-; CI-NEXT: s_mov_b32 s2, 0
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; CI-NEXT: s_endpgm
@@ -615,39 +616,40 @@ define amdgpu_kernel void @unaligned_read2_f32(ptr addrspace(1) %out, ptr addrsp
define amdgpu_kernel void @unaligned_offset_read2_f32(ptr addrspace(1) %out, ptr addrspace(3) %lds) #0 {
; CI-LABEL: unaligned_offset_read2_f32:
; CI: ; %bb.0:
-; CI-NEXT: s_load_dword s2, s[4:5], 0x2
-; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CI-NEXT: s_load_dword s0, s[4:5], 0x2
; CI-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_mov_b32 s3, 0xf000
+; CI-NEXT: s_mov_b32 s2, 0
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_add_i32_e32 v1, vcc, s2, v0
+; CI-NEXT: v_add_i32_e32 v1, vcc, s0, v0
; CI-NEXT: ds_read_u8 v2, v1 offset:6
-; CI-NEXT: ds_read_u8 v3, v1 offset:11
-; CI-NEXT: ds_read_u8 v4, v1 offset:9
+; CI-NEXT: ds_read_u8 v3, v1 offset:5
+; CI-NEXT: ds_read_u8 v4, v1 offset:8
; CI-NEXT: ds_read_u8 v5, v1 offset:7
-; CI-NEXT: ds_read_u8 v6, v1 offset:5
-; CI-NEXT: ds_read_u8 v7, v1 offset:8
-; CI-NEXT: ds_read_u8 v8, v1 offset:10
-; CI-NEXT: ds_read_u8 v1, v1 offset:12
-; CI-NEXT: s_waitcnt lgkmcnt(7)
+; CI-NEXT: ds_read_u8 v6, v1 offset:12
+; CI-NEXT: s_waitcnt lgkmcnt(4)
; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
; CI-NEXT: s_waitcnt lgkmcnt(3)
-; CI-NEXT: v_or_b32_e32 v2, v2, v6
+; CI-NEXT: v_or_b32_e32 v2, v2, v3
+; CI-NEXT: s_waitcnt lgkmcnt(2)
+; CI-NEXT: v_lshlrev_b32_e32 v3, 8, v4
+; CI-NEXT: ds_read_u8 v4, v1 offset:10
; CI-NEXT: s_waitcnt lgkmcnt(2)
-; CI-NEXT: v_lshlrev_b32_e32 v6, 8, v7
-; CI-NEXT: v_or_b32_e32 v5, v6, v5
-; CI-NEXT: v_lshlrev_b32_e32 v5, 16, v5
+; CI-NEXT: v_or_b32_e32 v3, v3, v5
+; CI-NEXT: ds_read_u8 v5, v1 offset:9
+; CI-NEXT: ds_read_u8 v1, v1 offset:11
+; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CI-NEXT: v_or_b32_e32 v2, v3, v2
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; CI-NEXT: v_or_b32_e32 v2, v5, v2
-; CI-NEXT: v_lshlrev_b32_e32 v5, 8, v8
-; CI-NEXT: v_or_b32_e32 v1, v1, v3
-; CI-NEXT: v_or_b32_e32 v4, v5, v4
+; CI-NEXT: v_lshlrev_b32_e32 v3, 8, v4
+; CI-NEXT: v_lshlrev_b32_e32 v4, 8, v6
+; CI-NEXT: v_or_b32_e32 v1, v4, v1
+; CI-NEXT: v_or_b32_e32 v3, v3, v5
; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_or_b32_e32 v1, v1, v4
+; CI-NEXT: v_or_b32_e32 v1, v1, v3
; CI-NEXT: v_add_f32_e32 v2, v2, v1
-; CI-NEXT: s_mov_b32 s2, 0
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
; CI-NEXT: s_endpgm
@@ -716,15 +718,15 @@ define amdgpu_kernel void @misaligned_2_simple_read2_f32(ptr addrspace(1) %out,
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_add_i32_e32 v1, vcc, s0, v0
; CI-NEXT: ds_read_u16 v2, v1 offset:2
-; CI-NEXT: ds_read_u16 v3, v1 offset:32
-; CI-NEXT: ds_read_u16 v4, v1
-; CI-NEXT: ds_read_u16 v1, v1 offset:34
+; CI-NEXT: ds_read_u16 v3, v1
+; CI-NEXT: ds_read_u16 v4, v1 offset:34
+; CI-NEXT: ds_read_u16 v1, v1 offset:32
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; CI-NEXT: v_or_b32_e32 v2, v2, v4
-; CI-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; CI-NEXT: v_or_b32_e32 v1, v1, v3
+; CI-NEXT: v_or_b32_e32 v2, v2, v3
+; CI-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; CI-NEXT: v_or_b32_e32 v1, v3, v1
; CI-NEXT: v_add_f32_e32 v2, v2, v1
; CI-NEXT: v_mov_b32_e32 v1, 0
; CI-NEXT: buffer_store_dword v2, v[0:1], s[0:3], 0 addr64
@@ -1068,24 +1070,25 @@ define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(ptr addrspace(1) %C,
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; CI-NEXT: s_lshl_b32 s4, s8, 2
; CI-NEXT: s_add_i32 s5, s4, 0xc20
-; CI-NEXT: s_addk_i32 s4, 0xc60
; CI-NEXT: v_mov_b32_e32 v0, s5
-; CI-NEXT: v_mov_b32_e32 v2, s4
-; CI-NEXT: v_lshlrev_b32_e32 v8, 2, v1
; CI-NEXT: s_mov_b32 m0, -1
-; CI-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
-; CI-NEXT: ds_read2_b32 v[2:3], v2 offset1:1
-; CI-NEXT: ds_read2_b32 v[4:5], v8 offset1:1
+; CI-NEXT: s_addk_i32 s4, 0xc60
+; CI-NEXT: ds_read2_b32 v[2:3], v0 offset1:1
+; CI-NEXT: v_mov_b32_e32 v0, s4
+; CI-NEXT: ds_read2_b32 v[4:5], v0 offset1:1
+; CI-NEXT: v_lshlrev_b32_e32 v8, 2, v1
+; CI-NEXT: ds_read2_b32 v[0:1], v8 offset1:1
; CI-NEXT: ds_read2_b32 v[6:7], v8 offset0:32 offset1:33
-; CI-NEXT: ds_read2_b32 v[8:9], v8 offset0:64 offset1:65
; CI-NEXT: s_waitcnt lgkmcnt(0)
+; CI-NEXT: v_add_f32_e32 v2, v2, v3
+; CI-NEXT: v_add_f32_e32 v2, v2, v4
+; CI-NEXT: ds_read2_b32 v[8:9], v8 offset0:64 offset1:65
+; CI-NEXT: v_add_f32_e32 v2, v2, v5
+; CI-NEXT: v_add_f32_e32 v0, v2, v0
; CI-NEXT: v_add_f32_e32 v0, v0, v1
-; CI-NEXT: v_add_f32_e32 v0, v0, v2
-; CI-NEXT: v_add_f32_e32 v0, v0, v3
-; CI-NEXT: v_add_f32_e32 v0, v0, v4
-; CI-NEXT: v_add_f32_e32 v0, v0, v5
; CI-NEXT: v_add_f32_e32 v0, v0, v6
; CI-NEXT: v_add_f32_e32 v0, v0, v7
+; CI-NEXT: s_waitcnt lgkmcnt(0)
; CI-NEXT: v_add_f32_e32 v0, v0, v8
; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
@@ -1097,24 +1100,24 @@ define amdgpu_kernel void @sgemm_inner_loop_read2_sequence(ptr addrspace(1) %C,
; GFX9: ; %bb.0:
; GFX9-NEXT: s_lshl_b32 s2, s8, 2
; GFX9-NEXT: s_add_i32 s3, s2, 0xc20
-; GFX9-NEXT: s_addk_i32 s2, 0xc60
; GFX9-NEXT: v_mov_b32_e32 v0, s3
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
+; GFX9-NEXT: s_addk_i32 s2, 0xc60
+; GFX9-NEXT: ds_read2_b32 v[2:3], v0 offset1:1
+; GFX9-NEXT: v_mov_b32_e32 v0, s2
+; GFX9-NEXT: ds_read2_b32 v[4:5], v0 offset1:1
; GFX9-NEXT: v_lshlrev_b32_e32 v8, 2, v1
-; GFX9-NEXT: ds_read2_b32 v[0:1], v0 offset1:1
-; GFX9-NEXT: ds_read2_b32 v[2:3], v2 offset1:1
-; GFX9-NEXT: ds_read2_b32 v[4:5], v8 offset1:1
+; GFX9-NEXT: ds_read2_b32 v[0:1], v8 offset1:1
; GFX9-NEXT: ds_read2_b32 v[6:7], v8 offset0:32 offset1:33
-; GFX9-NEXT: ds_read2_b32 v[8:9], v8 offset0:64 offset1:65
-; GFX9-NEXT: s_waitcnt lgkmcnt(4)
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(3)
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v2
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v3
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v3
; GFX9-NEXT: s_waitcnt lgkmcnt(2)
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v4
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v4
+; GFX9-NEXT: ds_read2_b32 v[8:9], v8 offset0:64 offset1:65
+; GFX9-NEXT: v_add_f32_e32 v2, v2, v5
+; GFX9-NEXT: s_waitcnt lgkmcnt(2)
+; GFX9-NEXT: v_add_f32_e32 v0, v2, v0
; GFX9-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
-; GFX9-NEXT: v_add_f32_e32 v0, v0, v5
+; GFX9-NEXT: v_add_f32_e32 v0, v0, v1
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_add_f32_e32 v0, v0, v6
; GFX9-NEXT: v_add_f32_e32 v0, v0, v7
@@ -1454,58 +1457,58 @@ define amdgpu_kernel void @read2_v2i32_align1_odd_offset(ptr addrspace(1) %out)
; CI-NEXT: v_mov_b32_e32 v0, 0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: ds_read_u8 v1, v0 offset:70
-; CI-NEXT: ds_read_u8 v2, v0 offset:72
-; CI-NEXT: ds_read_u8 v3, v0 offset:71
-; CI-NEXT: ds_read_u8 v4, v0 offset:69
+; CI-NEXT: ds_read_u8 v2, v0 offset:69
+; CI-NEXT: ds_read_u8 v3, v0 offset:72
+; CI-NEXT: ds_read_u8 v4, v0 offset:71
; CI-NEXT: ds_read_u8 v5, v0 offset:68
; CI-NEXT: s_waitcnt lgkmcnt(4)
; CI-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; CI-NEXT: s_waitcnt lgkmcnt(3)
-; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v2
+; CI-NEXT: v_or_b32_e32 v1, v1, v2
; CI-NEXT: s_waitcnt lgkmcnt(2)
-; CI-NEXT: v_or_b32_e32 v2, v2, v3
-; CI-NEXT: s_waitcnt lgkmcnt(1)
-; CI-NEXT: v_or_b32_e32 v1, v1, v4
-; CI-NEXT: ds_read_u8 v4, v0 offset:66
-; CI-NEXT: ds_read_u8 v6, v0 offset:67
-; CI-NEXT: ds_read_u8 v0, v0 offset:65
-; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v3
+; CI-NEXT: ds_read_u8 v3, v0 offset:66
+; CI-NEXT: s_waitcnt lgkmcnt(2)
+; CI-NEXT: v_or_b32_e32 v2, v2, v4
+; CI-NEXT: ds_read_u8 v4, v0 offset:65
+; CI-NEXT: ds_read_u8 v0, v0 offset:67
; CI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
+; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; CI-NEXT: v_or_b32_e32 v1, v2, v1
; CI-NEXT: s_waitcnt lgkmcnt(0)
-; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v4
-; CI-NEXT: v_or_b32_e32 v0, v2, v0
-; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v5
-; CI-NEXT: v_or_b32_e32 v2, v2, v6
-; CI-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; CI-NEXT: v_lshlrev_b32_e32 v2, 8, v3
+; CI-NEXT: v_lshlrev_b32_e32 v3, 8, v5
+; CI-NEXT: v_or_b32_e32 v0, v3, v0
+; CI-NEXT: v_or_b32_e32 v2, v2, v4
+; CI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; CI-NEXT: s_mov_b32 s3, 0xf000
; CI-NEXT: s_mov_b32 s2, -1
-; CI-NEXT: v_or_b32_e32 v0, v2, v0
+; CI-NEXT: v_or_b32_e32 v0, v0, v2
; CI-NEXT: buffer_store_dwordx2 v[0:1], off, s[0:3], 0
; CI-NEXT: s_endpgm
;
; GFX9-ALIGNED-LABEL: read2_v2i32_align1_odd_offset:
; GFX9-ALIGNED: ; %bb.0: ; %entry
; GFX9-ALIGNED-NEXT: v_mov_b32_e32 v2, 0
-; GFX9-ALIGNED-NEXT: ds_read_u8 v0, v2 offset:65
; GFX9-ALIGNED-NEXT: ds_read_u8 v3, v2 offset:66
-; GFX9-ALIGNED-NEXT: ds_read_u8 v4, v2 offset:67
-; GFX9-ALIGNED-NEXT: ds_read_u8 v5, v2 offset:68
; GFX9-ALIGNED-NEXT: ds_read_u8 v1, v2 offset:70
-; GFX9-ALIGNED-NEXT: ds_read_u8 v6, v2 offset:69
-; GFX9-ALIGNED-NEXT: ds_read_u8 v7, v2 offset:72
-; GFX9-ALIGNED-NEXT: ds_read_u8 v8, v2 offset:71
+; GFX9-ALIGNED-NEXT: ds_read_u8 v0, v2 offset:65
+; GFX9-ALIGNED-NEXT: ds_read_u8 v5, v2 offset:69
+; GFX9-ALIGNED-NEXT: ds_read_u8 v6, v2 offset:72
+; GFX9-ALIGNED-NEXT: ds_read_u8 v8, v2 offset:68
+; GFX9-ALIGNED-NEXT: ds_read_u8 v4, v2 offset:67
+; GFX9-ALIGNED-NEXT: ds_read_u8 v7, v2 offset:71
; GFX9-ALIGNED-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x0
; GFX9-ALIGNED-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v3, 8, v3
-; GFX9-ALIGNED-NEXT: v_or_b32_e32 v1, v1, v6
-; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v6, 8, v7
+; GFX9-ALIGNED-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v5, 8, v6
; GFX9-ALIGNED-NEXT: v_or_b32_e32 v0, v3, v0
-; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v3, 8, v5
-; GFX9-ALIGNED-NEXT: v_or_b32_sdwa v6, v6, v8 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
+; GFX9-ALIGNED-NEXT: v_lshlrev_b32_e32 v3, 8, v8
+; GFX9-ALIGNED-NEXT: v_or_b32_sdwa v5, v5, v7 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
; GFX9-ALIGNED-NEXT: v_or_b32_sdwa v3, v3, v4 dst_sel:WORD_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:DWORD
-; GFX9-ALIGNED-NEXT: v_or_b32_e32 v1, v6, v1
+; GFX9-ALIGNED-NEXT: v_or_b32_e32 v1, v5, v1
; GFX9-ALIGNED-NEXT: v_or_b32_e32 v0, v3, v0
; GFX9-ALIGNED-NEXT: global_store_dwordx2 v2, v[0:1], s[0:1]
; GFX9-ALIGNED-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/ds_write2.ll b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
index 1684437eff580..83d7fbe4f5311 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_write2.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_write2.ll
@@ -35,7 +35,7 @@ define amdgpu_kernel void @simple_write2_one_val_f32(ptr addrspace(1) %C, ptr ad
; GFX1250-LABEL: simple_write2_one_val_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
@@ -87,7 +87,7 @@ define amdgpu_kernel void @simple_write2_two_val_f32(ptr addrspace(1) %C, ptr ad
; GFX1250-LABEL: simple_write2_two_val_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
@@ -148,7 +148,7 @@ define amdgpu_kernel void @simple_write2_two_val_f32_volatile_0(ptr addrspace(1)
; GFX1250-LABEL: simple_write2_two_val_f32_volatile_0:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
@@ -210,7 +210,7 @@ define amdgpu_kernel void @simple_write2_two_val_f32_volatile_1(ptr addrspace(1)
; GFX1250-LABEL: simple_write2_two_val_f32_volatile_1:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
@@ -276,7 +276,7 @@ define amdgpu_kernel void @simple_write2_two_val_subreg2_mixed_f32(ptr addrspace
; GFX1250-LABEL: simple_write2_two_val_subreg2_mixed_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b64 v[0:1], v4, s[0:1] scale_offset scope:SCOPE_SYS
@@ -331,7 +331,7 @@ define amdgpu_kernel void @simple_write2_two_val_subreg2_f32(ptr addrspace(1) %C
; GFX1250-LABEL: simple_write2_two_val_subreg2_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_and_b32_e32 v2, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b64 v[0:1], v2, s[0:1] scale_offset
@@ -383,7 +383,7 @@ define amdgpu_kernel void @simple_write2_two_val_subreg4_f32(ptr addrspace(1) %C
; GFX1250-LABEL: simple_write2_two_val_subreg4_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_and_b32_e32 v4, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_b128 v[0:3], v4, s[0:1] scale_offset
@@ -436,7 +436,7 @@ define amdgpu_kernel void @simple_write2_two_val_max_offset_f32(ptr addrspace(1)
; GFX1250-LABEL: simple_write2_two_val_max_offset_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
@@ -497,7 +497,7 @@ define amdgpu_kernel void @simple_write2_two_val_too_far_f32(ptr addrspace(1) %C
; GFX1250-LABEL: simple_write2_two_val_too_far_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
@@ -558,7 +558,7 @@ define amdgpu_kernel void @simple_write2_two_val_f32_x2(ptr addrspace(1) %C, ptr
; GFX1250-LABEL: simple_write2_two_val_f32_x2:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
@@ -630,7 +630,7 @@ define amdgpu_kernel void @simple_write2_two_val_f32_x2_nonzero_base(ptr addrspa
; GFX1250-LABEL: simple_write2_two_val_f32_x2_nonzero_base:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffc, v0
@@ -683,8 +683,8 @@ define amdgpu_kernel void @write2_ptr_subreg_arg_two_val_f32(ptr addrspace(1) %C
; CI-NEXT: buffer_load_dword v2, v[0:1], s[4:7], 0 addr64
; CI-NEXT: buffer_load_dword v0, v[0:1], s[0:3], 0 addr64
; CI-NEXT: v_mov_b32_e32 v1, s8
-; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: v_mov_b32_e32 v3, s9
+; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt vmcnt(1)
; CI-NEXT: ds_write_b32 v1, v2 offset:32
; CI-NEXT: s_waitcnt vmcnt(0)
@@ -710,10 +710,10 @@ define amdgpu_kernel void @write2_ptr_subreg_arg_two_val_f32(ptr addrspace(1) %C
; GFX1250-LABEL: write2_ptr_subreg_arg_two_val_f32:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x8 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_xcnt 0x0
-; GFX1250-NEXT: s_load_b64 s[4:5], s[4:5], 0x18
+; GFX1250-NEXT: s_load_b64 s[4:5], s[4:5], 0x18 nv
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_load_b32 v1, v0, s[0:1] scale_offset
@@ -774,7 +774,7 @@ define amdgpu_kernel void @simple_write2_one_val_f64(ptr addrspace(1) %C, ptr ad
; GFX1250-LABEL: simple_write2_one_val_f64:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v2, 0x1ff8, v0
@@ -828,7 +828,7 @@ define amdgpu_kernel void @misaligned_simple_write2_one_val_f64(ptr addrspace(1)
; GFX1250-LABEL: misaligned_simple_write2_one_val_f64:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v2, 0x1ff8, v0
@@ -865,12 +865,12 @@ define amdgpu_kernel void @unaligned_offset_simple_write2_one_val_f64(ptr addrsp
; CI-NEXT: v_add_i32_e32 v0, vcc, s4, v0
; CI-NEXT: s_mov_b32 m0, -1
; CI-NEXT: s_waitcnt vmcnt(0)
-; CI-NEXT: ds_write_b8 v0, v1 offset:9
-; CI-NEXT: ds_write_b8 v0, v2 offset:13
-; CI-NEXT: v_lshrrev_b32_e32 v3, 24, v1
; CI-NEXT: ds_write_b8 v0, v1 offset:5
+; CI-NEXT: v_lshrrev_b32_e32 v3, 24, v1
; CI-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; CI-NEXT: v_lshrrev_b32_e32 v5, 8, v1
+; CI-NEXT: ds_write_b8 v0, v1 offset:9
+; CI-NEXT: ds_write_b8 v0, v2 offset:13
; CI-NEXT: v_lshrrev_b32_e32 v1, 24, v2
; CI-NEXT: v_lshrrev_b32_e32 v6, 16, v2
; CI-NEXT: v_lshrrev_b32_e32 v2, 8, v2
@@ -928,7 +928,7 @@ define amdgpu_kernel void @unaligned_offset_simple_write2_one_val_f64(ptr addrsp
; GFX1250-LABEL: unaligned_offset_simple_write2_one_val_f64:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b96 s[0:2], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v2, 0x1ff8, v0
@@ -983,7 +983,7 @@ define amdgpu_kernel void @simple_write2_two_val_f64(ptr addrspace(1) %C, ptr ad
; GFX1250-LABEL: simple_write2_two_val_f64:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x8 nv
; GFX1250-NEXT: v_lshlrev_b32_e32 v0, 3, v0
; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1250-NEXT: v_and_b32_e32 v4, 0x1ff8, v0
@@ -1178,8 +1178,8 @@ define amdgpu_kernel void @write2_sgemm_sequence(ptr addrspace(1) %C, i32 %lda,
; GFX9-NEXT: v_mov_b32_e32 v3, s0
; GFX9-NEXT: v_mov_b32_e32 v4, s0
; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset1:1
-; GFX9-NEXT: ds_write2_b32 v2, v3, v4 offset1:1
; GFX9-NEXT: v_lshlrev_b32_e32 v0, 2, v1
+; GFX9-NEXT: ds_write2_b32 v2, v3, v4 offset1:1
; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset1:1
; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset0:32 offset1:33
; GFX9-NEXT: ds_write2_b32 v0, v3, v4 offset0:64 offset1:65
@@ -1188,7 +1188,7 @@ define amdgpu_kernel void @write2_sgemm_sequence(ptr addrspace(1) %C, i32 %lda,
; GFX1250-LABEL: write2_sgemm_sequence:
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
-; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x10
+; GFX1250-NEXT: s_load_b64 s[0:1], s[4:5], 0x10 nv
; GFX1250-NEXT: s_and_b32 s2, ttmp6, 15
; GFX1250-NEXT: s_getreg_b32 s3, hwreg(HW_REG_IB_STS2, 6, 4)
; GFX1250-NEXT: s_wait_kmcnt 0x0
@@ -1306,8 +1306,8 @@ define amdgpu_kernel void @simple_write2_v4f32_superreg_align4(ptr addrspace(3)
; GFX1250: ; %bb.0:
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1
; GFX1250-NEXT: s_clause 0x1
-; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x8
-; GFX1250-NEXT: s_load_b32 s8, s[4:5], 0x0
+; GFX1250-NEXT: s_load_b64 s[6:7], s[4:5], 0x8 nv
+; GFX1250-NEXT: s_load_b32 s8, s[4:5], 0x0 nv
; GFX1250-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: s_load_b128 s[0:3], s[6:7], 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/ds_write2_a_v.ll b/llvm/test/CodeGen/AMDGPU/ds_write2_a_v.ll
index 683887b0a55f3..151d6b4c334fc 100644
--- a/llvm/test/CodeGen/AMDGPU/ds_write2_a_v.ll
+++ b/llvm/test/CodeGen/AMDGPU/ds_write2_a_v.ll
@@ -61,10 +61,10 @@ define void @ds_write2_b32_v_a(ptr addrspace(3) %lds) #0 {
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: ;;#ASMSTART
-; GCN-NEXT: ; def a0
+; GCN-NEXT: ; def v1
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ;;#ASMSTART
-; GCN-NEXT: ; def v1
+; GCN-NEXT: ; def a0
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ds_write_b32 v0, a0 offset:40
; GCN-NEXT: ds_write_b32 v0, v1 offset:96
@@ -268,10 +268,10 @@ define void @ds_write2st64_b32_v_a(ptr addrspace(3) %lds) #0 {
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: ;;#ASMSTART
-; GCN-NEXT: ; def v1
+; GCN-NEXT: ; def a0
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ;;#ASMSTART
-; GCN-NEXT: ; def a0
+; GCN-NEXT: ; def v1
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ds_write_b32 v0, v1 offset:256
; GCN-NEXT: ds_write_b32 v0, a0 offset:1024
@@ -612,10 +612,10 @@ define void @ds_write2_b64_v_a(ptr addrspace(3) %lds) #0 {
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: ;;#ASMSTART
-; GCN-NEXT: ; def a[0:1]
+; GCN-NEXT: ; def v[2:3]
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ;;#ASMSTART
-; GCN-NEXT: ; def v[2:3]
+; GCN-NEXT: ; def a[0:1]
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ds_write_b64 v0, a[0:1] offset:80
; GCN-NEXT: ds_write_b64 v0, v[2:3] offset:192
@@ -819,10 +819,10 @@ define void @ds_write2st64_b64_v_a(ptr addrspace(3) %lds) #0 {
; GCN: ; %bb.0:
; GCN-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GCN-NEXT: ;;#ASMSTART
-; GCN-NEXT: ; def v[2:3]
+; GCN-NEXT: ; def a[0:1]
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ;;#ASMSTART
-; GCN-NEXT: ; def a[0:1]
+; GCN-NEXT: ; def v[2:3]
; GCN-NEXT: ;;#ASMEND
; GCN-NEXT: ds_write_b64 v0, v[2:3] offset:512
; GCN-NEXT: ds_write_b64 v0, a[0:1] offset:2048
diff --git a/llvm/test/CodeGen/AMDGPU/expand-waitcnt-profiling.ll b/llvm/test/CodeGen/AMDGPU/expand-waitcnt-profiling.ll
index 07005fd020e81..ca5efe7d41fdb 100644
--- a/llvm/test/CodeGen/AMDGPU/expand-waitcnt-profiling.ll
+++ b/llvm/test/CodeGen/AMDGPU/expand-waitcnt-profiling.ll
@@ -374,12 +374,12 @@ define amdgpu_kernel void @test_lgkmcnt_lds_operations(ptr addrspace(3) %lds_ptr
; GFX11-EXPAND-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-EXPAND-NEXT: v_mov_b32_e32 v2, s2
; GFX11-EXPAND-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:1
-; GFX11-EXPAND-NEXT: ds_load_b32 v2, v2 offset:8
-; GFX11-EXPAND-NEXT: s_waitcnt lgkmcnt(1)
+; GFX11-EXPAND-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-EXPAND-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX11-EXPAND-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-EXPAND-NEXT: ds_load_b32 v2, v2 offset:8
; GFX11-EXPAND-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-EXPAND-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-EXPAND-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, v0, v2
+; GFX11-EXPAND-NEXT: v_add_nc_u32_e32 v0, v0, v2
; GFX11-EXPAND-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-EXPAND-NEXT: s_endpgm
;
@@ -391,12 +391,12 @@ define amdgpu_kernel void @test_lgkmcnt_lds_operations(ptr addrspace(3) %lds_ptr
; GFX11-NOEXPAND-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NOEXPAND-NEXT: v_mov_b32_e32 v2, s2
; GFX11-NOEXPAND-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:1
-; GFX11-NOEXPAND-NEXT: ds_load_b32 v2, v2 offset:8
-; GFX11-NOEXPAND-NEXT: s_waitcnt lgkmcnt(1)
+; GFX11-NOEXPAND-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NOEXPAND-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX11-NOEXPAND-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-NOEXPAND-NEXT: ds_load_b32 v2, v2 offset:8
; GFX11-NOEXPAND-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NOEXPAND-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-NOEXPAND-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, v0, v2
+; GFX11-NOEXPAND-NEXT: v_add_nc_u32_e32 v0, v0, v2
; GFX11-NOEXPAND-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX11-NOEXPAND-NEXT: s_endpgm
;
@@ -408,12 +408,12 @@ define amdgpu_kernel void @test_lgkmcnt_lds_operations(ptr addrspace(3) %lds_ptr
; GFX12-EXPAND-NEXT: s_wait_kmcnt 0x0
; GFX12-EXPAND-NEXT: v_mov_b32_e32 v2, s2
; GFX12-EXPAND-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:1
-; GFX12-EXPAND-NEXT: ds_load_b32 v2, v2 offset:8
-; GFX12-EXPAND-NEXT: s_wait_dscnt 0x1
+; GFX12-EXPAND-NEXT: s_wait_dscnt 0x0
; GFX12-EXPAND-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-EXPAND-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-EXPAND-NEXT: ds_load_b32 v2, v2 offset:8
; GFX12-EXPAND-NEXT: s_wait_dscnt 0x0
-; GFX12-EXPAND-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-EXPAND-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, v0, v2
+; GFX12-EXPAND-NEXT: v_add_nc_u32_e32 v0, v0, v2
; GFX12-EXPAND-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-EXPAND-NEXT: s_endpgm
;
@@ -425,12 +425,12 @@ define amdgpu_kernel void @test_lgkmcnt_lds_operations(ptr addrspace(3) %lds_ptr
; GFX12-NOEXPAND-NEXT: s_wait_kmcnt 0x0
; GFX12-NOEXPAND-NEXT: v_mov_b32_e32 v2, s2
; GFX12-NOEXPAND-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:1
-; GFX12-NOEXPAND-NEXT: ds_load_b32 v2, v2 offset:8
-; GFX12-NOEXPAND-NEXT: s_wait_dscnt 0x1
+; GFX12-NOEXPAND-NEXT: s_wait_dscnt 0x0
; GFX12-NOEXPAND-NEXT: v_add_nc_u32_e32 v0, v0, v1
+; GFX12-NOEXPAND-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NOEXPAND-NEXT: ds_load_b32 v2, v2 offset:8
; GFX12-NOEXPAND-NEXT: s_wait_dscnt 0x0
-; GFX12-NOEXPAND-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NOEXPAND-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_add_nc_u32 v0, v0, v2
+; GFX12-NOEXPAND-NEXT: v_add_nc_u32_e32 v0, v0, v2
; GFX12-NOEXPAND-NEXT: global_store_b32 v1, v0, s[0:1]
; GFX12-NOEXPAND-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
index d879ebede164e..23b8f194b1682 100644
--- a/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
+++ b/llvm/test/CodeGen/AMDGPU/extract-subvector-16bit.ll
@@ -1308,15 +1308,15 @@ define <8 x i16> @large_vector(ptr addrspace(3) %p, i32 %idxp) {
; SI: ; %bb.0:
; SI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; SI-NEXT: v_lshlrev_b32_e32 v1, 5, v1
-; SI-NEXT: v_add_i32_e32 v0, vcc, v0, v1
-; SI-NEXT: v_add_i32_e32 v1, vcc, 4, v0
-; SI-NEXT: v_add_i32_e32 v2, vcc, 8, v0
-; SI-NEXT: v_add_i32_e32 v3, vcc, 12, v0
+; SI-NEXT: v_add_i32_e32 v1, vcc, v0, v1
; SI-NEXT: s_mov_b32 m0, -1
-; SI-NEXT: ds_read_b32 v0, v0
-; SI-NEXT: ds_read_b32 v1, v1
-; SI-NEXT: ds_read_b32 v2, v2
-; SI-NEXT: ds_read_b32 v3, v3
+; SI-NEXT: ds_read_b32 v0, v1
+; SI-NEXT: v_add_i32_e32 v2, vcc, 4, v1
+; SI-NEXT: v_add_i32_e32 v3, vcc, 8, v1
+; SI-NEXT: v_add_i32_e32 v4, vcc, 12, v1
+; SI-NEXT: ds_read_b32 v1, v2
+; SI-NEXT: ds_read_b32 v2, v3
+; SI-NEXT: ds_read_b32 v3, v4
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/freeze.ll b/llvm/test/CodeGen/AMDGPU/freeze.ll
index 6ed4e6aed92d8..8fed64643aa5e 100644
--- a/llvm/test/CodeGen/AMDGPU/freeze.ll
+++ b/llvm/test/CodeGen/AMDGPU/freeze.ll
@@ -10380,24 +10380,24 @@ define void @freeze_v8p3(ptr addrspace(3) %ptra, ptr addrspace(3) %ptrb) {
; GFX6-SDAG-LABEL: freeze_v8p3:
; GFX6-SDAG: ; %bb.0:
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-SDAG-NEXT: v_add_i32_e32 v2, vcc, 24, v0
-; GFX6-SDAG-NEXT: v_add_i32_e32 v4, vcc, 16, v0
; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
-; GFX6-SDAG-NEXT: ds_read_b64 v[2:3], v2
-; GFX6-SDAG-NEXT: ds_read_b64 v[4:5], v4
+; GFX6-SDAG-NEXT: v_add_i32_e32 v2, vcc, 8, v0
+; GFX6-SDAG-NEXT: v_add_i32_e32 v8, vcc, 24, v0
+; GFX6-SDAG-NEXT: ds_read_b64 v[4:5], v0
+; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 16, v0
; GFX6-SDAG-NEXT: ds_read_b64 v[6:7], v0
-; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 8, v0
-; GFX6-SDAG-NEXT: ds_read_b64 v[8:9], v0
+; GFX6-SDAG-NEXT: ds_read_b64 v[8:9], v8
+; GFX6-SDAG-NEXT: ds_read_b64 v[2:3], v2
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 16, v1
; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(2)
-; GFX6-SDAG-NEXT: ds_write_b64 v0, v[4:5]
+; GFX6-SDAG-NEXT: ds_write_b64 v0, v[6:7]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 24, v1
-; GFX6-SDAG-NEXT: ds_write_b64 v0, v[2:3]
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-SDAG-NEXT: ds_write_b64 v0, v[8:9]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 8, v1
+; GFX6-SDAG-NEXT: ds_write_b64 v1, v[4:5]
; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(3)
-; GFX6-SDAG-NEXT: ds_write_b64 v1, v[6:7]
-; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(3)
-; GFX6-SDAG-NEXT: ds_write_b64 v0, v[8:9]
+; GFX6-SDAG-NEXT: ds_write_b64 v0, v[2:3]
; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
;
@@ -10406,14 +10406,14 @@ define void @freeze_v8p3(ptr addrspace(3) %ptra, ptr addrspace(3) %ptrb) {
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
; GFX6-GISEL-NEXT: v_add_i32_e32 v4, vcc, 8, v0
+; GFX6-GISEL-NEXT: ds_read_b64 v[4:5], v4
; GFX6-GISEL-NEXT: v_add_i32_e32 v6, vcc, 16, v0
; GFX6-GISEL-NEXT: ds_read_b64 v[2:3], v0
-; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, 24, v0
-; GFX6-GISEL-NEXT: ds_read_b64 v[4:5], v4
; GFX6-GISEL-NEXT: ds_read_b64 v[6:7], v6
+; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, 24, v0
; GFX6-GISEL-NEXT: ds_read_b64 v[8:9], v0
; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, 8, v1
-; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(3)
; GFX6-GISEL-NEXT: ds_write_b64 v0, v[4:5]
; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, 16, v1
; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(2)
@@ -10533,39 +10533,42 @@ define void @freeze_v16p3(ptr addrspace(3) %ptra, ptr addrspace(3) %ptrb) {
; GFX6-SDAG-LABEL: freeze_v16p3:
; GFX6-SDAG: ; %bb.0:
; GFX6-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
; GFX6-SDAG-NEXT: v_add_i32_e32 v2, vcc, 8, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v4, vcc, 24, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v6, vcc, 16, v0
-; GFX6-SDAG-NEXT: v_add_i32_e32 v10, vcc, 40, v0
-; GFX6-SDAG-NEXT: v_add_i32_e32 v12, vcc, 32, v0
+; GFX6-SDAG-NEXT: v_add_i32_e32 v8, vcc, 40, v0
+; GFX6-SDAG-NEXT: v_add_i32_e32 v10, vcc, 32, v0
; GFX6-SDAG-NEXT: v_add_i32_e32 v16, vcc, 56, v0
-; GFX6-SDAG-NEXT: v_add_i32_e32 v14, vcc, 48, v0
-; GFX6-SDAG-NEXT: s_mov_b32 m0, -1
-; GFX6-SDAG-NEXT: ds_read_b64 v[2:3], v2
-; GFX6-SDAG-NEXT: ds_read_b64 v[4:5], v4
-; GFX6-SDAG-NEXT: ds_read_b64 v[6:7], v6
-; GFX6-SDAG-NEXT: ds_read_b64 v[8:9], v0
-; GFX6-SDAG-NEXT: ds_read_b64 v[10:11], v10
-; GFX6-SDAG-NEXT: ds_read_b64 v[12:13], v12
-; GFX6-SDAG-NEXT: ds_read_b64 v[14:15], v14
+; GFX6-SDAG-NEXT: ds_read_b64 v[12:13], v0
+; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 48, v0
+; GFX6-SDAG-NEXT: ds_read_b64 v[14:15], v0
; GFX6-SDAG-NEXT: ds_read_b64 v[16:17], v16
+; GFX6-SDAG-NEXT: ds_read_b64 v[10:11], v10
+; GFX6-SDAG-NEXT: ds_read_b64 v[8:9], v8
+; GFX6-SDAG-NEXT: ds_read_b64 v[6:7], v6
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 48, v1
-; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(4)
-; GFX6-SDAG-NEXT: ds_write_b64 v1, v[8:9]
-; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-SDAG-NEXT: ds_read_b64 v[2:3], v2
+; GFX6-SDAG-NEXT: ds_read_b64 v[4:5], v4
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(6)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[14:15]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 56, v1
-; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(6)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[16:17]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 32, v1
-; GFX6-SDAG-NEXT: ds_write_b64 v0, v[12:13]
-; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 40, v1
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(6)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[10:11]
+; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 40, v1
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(6)
+; GFX6-SDAG-NEXT: ds_write_b64 v0, v[8:9]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 16, v1
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(6)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[6:7]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 24, v1
+; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(5)
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[4:5]
; GFX6-SDAG-NEXT: v_add_i32_e32 v0, vcc, 8, v1
+; GFX6-SDAG-NEXT: ds_write_b64 v1, v[12:13]
; GFX6-SDAG-NEXT: ds_write_b64 v0, v[2:3]
; GFX6-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-SDAG-NEXT: s_setpc_b64 s[30:31]
@@ -10573,30 +10576,30 @@ define void @freeze_v16p3(ptr addrspace(3) %ptra, ptr addrspace(3) %ptrb) {
; GFX6-GISEL-LABEL: freeze_v16p3:
; GFX6-GISEL: ; %bb.0:
; GFX6-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-GISEL-NEXT: v_add_i32_e32 v4, vcc, 8, v0
-; GFX6-GISEL-NEXT: v_add_i32_e32 v6, vcc, 16, v0
-; GFX6-GISEL-NEXT: v_add_i32_e32 v8, vcc, 24, v0
; GFX6-GISEL-NEXT: s_mov_b32 m0, -1
-; GFX6-GISEL-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-GISEL-NEXT: v_add_i32_e32 v4, vcc, 8, v0
; GFX6-GISEL-NEXT: ds_read_b64 v[4:5], v4
+; GFX6-GISEL-NEXT: v_add_i32_e32 v6, vcc, 16, v0
; GFX6-GISEL-NEXT: ds_read_b64 v[6:7], v6
+; GFX6-GISEL-NEXT: v_add_i32_e32 v8, vcc, 24, v0
; GFX6-GISEL-NEXT: ds_read_b64 v[8:9], v8
; GFX6-GISEL-NEXT: v_add_i32_e32 v10, vcc, 32, v0
+; GFX6-GISEL-NEXT: ds_read_b64 v[2:3], v0
+; GFX6-GISEL-NEXT: ds_read_b64 v[10:11], v10
; GFX6-GISEL-NEXT: v_add_i32_e32 v12, vcc, 40, v0
; GFX6-GISEL-NEXT: v_add_i32_e32 v14, vcc, 48, v0
; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, 56, v0
-; GFX6-GISEL-NEXT: ds_read_b64 v[10:11], v10
; GFX6-GISEL-NEXT: ds_read_b64 v[12:13], v12
-; GFX6-GISEL-NEXT: ds_read_b64 v[14:15], v14
; GFX6-GISEL-NEXT: ds_read_b64 v[16:17], v0
; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, 8, v1
-; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(6)
+; GFX6-GISEL-NEXT: ds_read_b64 v[14:15], v14
+; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(7)
; GFX6-GISEL-NEXT: ds_write_b64 v0, v[4:5]
; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, 16, v1
-; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(6)
+; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(7)
; GFX6-GISEL-NEXT: ds_write_b64 v0, v[6:7]
; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, 24, v1
-; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(6)
+; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(7)
; GFX6-GISEL-NEXT: ds_write_b64 v0, v[8:9]
; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, 32, v1
; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(6)
@@ -10605,11 +10608,10 @@ define void @freeze_v16p3(ptr addrspace(3) %ptra, ptr addrspace(3) %ptrb) {
; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(6)
; GFX6-GISEL-NEXT: ds_write_b64 v0, v[12:13]
; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, 48, v1
-; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(6)
+; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(5)
; GFX6-GISEL-NEXT: ds_write_b64 v0, v[14:15]
; GFX6-GISEL-NEXT: v_add_i32_e32 v0, vcc, 56, v1
; GFX6-GISEL-NEXT: ds_write_b64 v1, v[2:3]
-; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(7)
; GFX6-GISEL-NEXT: ds_write_b64 v0, v[16:17]
; GFX6-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-GISEL-NEXT: s_setpc_b64 s[30:31]
diff --git a/llvm/test/CodeGen/AMDGPU/lds-frame-extern.ll b/llvm/test/CodeGen/AMDGPU/lds-frame-extern.ll
index ad894ce36c55b..61461a3d5acd2 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-frame-extern.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-frame-extern.ll
@@ -96,10 +96,10 @@ define amdgpu_kernel void @module_0_kernel_normal_extern_normal(i32 %idx) {
; CHECK-NEXT: s_load_dword s0, s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 2
; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: ds_write_b16 v1, v0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_lshl2_add_u32 s0, s0, 4
; CHECK-NEXT: v_mov_b32_e32 v2, s0
-; CHECK-NEXT: ds_write_b16 v1, v0
; CHECK-NEXT: ds_write_b32 v2, v1
; CHECK-NEXT: s_endpgm
store i16 2, ptr addrspace(3) @kernel_normal
@@ -135,9 +135,9 @@ define amdgpu_kernel void @module_1_kernel_normal_extern_normal(i32 %idx) {
; CHECK-NEXT: v_or3_b32 v31, v0, v1, v2
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[20:21]
+; CHECK-NEXT: s_lshl2_add_u32 s4, s17, 4
; CHECK-NEXT: v_mov_b32_e32 v0, 1
; CHECK-NEXT: v_mov_b32_e32 v1, 0
-; CHECK-NEXT: s_lshl2_add_u32 s4, s17, 4
; CHECK-NEXT: v_mov_b32_e32 v2, 2
; CHECK-NEXT: v_mov_b32_e32 v3, s4
; CHECK-NEXT: ds_write_b16 v1, v0
@@ -160,10 +160,10 @@ define amdgpu_kernel void @module_0_kernel_overalign_extern_normal(i32 %idx) {
; CHECK-NEXT: s_load_dword s0, s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 2
; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: ds_write_b16 v1, v0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_lshl2_add_u32 s0, s0, 4
; CHECK-NEXT: v_mov_b32_e32 v2, s0
-; CHECK-NEXT: ds_write_b16 v1, v0
; CHECK-NEXT: ds_write_b32 v2, v1
; CHECK-NEXT: s_endpgm
store i16 2, ptr addrspace(3) @kernel_overalign
@@ -199,9 +199,9 @@ define amdgpu_kernel void @module_1_kernel_overalign_extern_normal(i32 %idx) {
; CHECK-NEXT: v_or3_b32 v31, v0, v1, v2
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[20:21]
+; CHECK-NEXT: s_lshl2_add_u32 s4, s17, 8
; CHECK-NEXT: v_mov_b32_e32 v0, 1
; CHECK-NEXT: v_mov_b32_e32 v1, 0
-; CHECK-NEXT: s_lshl2_add_u32 s4, s17, 8
; CHECK-NEXT: v_mov_b32_e32 v2, 2
; CHECK-NEXT: v_mov_b32_e32 v3, s4
; CHECK-NEXT: ds_write_b16 v1, v0
@@ -224,10 +224,10 @@ define amdgpu_kernel void @module_0_kernel_normal_extern_overalign(i32 %idx) {
; CHECK-NEXT: s_load_dword s0, s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 2
; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: ds_write_b16 v1, v0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_lshl2_add_u32 s0, s0, 8
; CHECK-NEXT: v_mov_b32_e32 v2, s0
-; CHECK-NEXT: ds_write_b16 v1, v0
; CHECK-NEXT: ds_write_b32 v2, v1
; CHECK-NEXT: s_endpgm
store i16 2, ptr addrspace(3) @kernel_normal
@@ -263,9 +263,9 @@ define amdgpu_kernel void @module_1_kernel_normal_extern_overalign(i32 %idx) {
; CHECK-NEXT: v_or3_b32 v31, v0, v1, v2
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[20:21]
+; CHECK-NEXT: s_lshl2_add_u32 s4, s17, 8
; CHECK-NEXT: v_mov_b32_e32 v0, 1
; CHECK-NEXT: v_mov_b32_e32 v1, 0
-; CHECK-NEXT: s_lshl2_add_u32 s4, s17, 8
; CHECK-NEXT: v_mov_b32_e32 v2, 2
; CHECK-NEXT: v_mov_b32_e32 v3, s4
; CHECK-NEXT: ds_write_b16 v1, v0
@@ -288,10 +288,10 @@ define amdgpu_kernel void @module_0_kernel_overalign_extern_overalign(i32 %idx)
; CHECK-NEXT: s_load_dword s0, s[8:9], 0x0
; CHECK-NEXT: v_mov_b32_e32 v0, 2
; CHECK-NEXT: v_mov_b32_e32 v1, 0
+; CHECK-NEXT: ds_write_b16 v1, v0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_lshl2_add_u32 s0, s0, 8
; CHECK-NEXT: v_mov_b32_e32 v2, s0
-; CHECK-NEXT: ds_write_b16 v1, v0
; CHECK-NEXT: ds_write_b32 v2, v1
; CHECK-NEXT: s_endpgm
store i16 2, ptr addrspace(3) @kernel_overalign
@@ -327,9 +327,9 @@ define amdgpu_kernel void @module_1_kernel_overalign_extern_overalign(i32 %idx)
; CHECK-NEXT: v_or3_b32 v31, v0, v1, v2
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_swappc_b64 s[30:31], s[20:21]
+; CHECK-NEXT: s_lshl2_add_u32 s4, s17, 8
; CHECK-NEXT: v_mov_b32_e32 v0, 1
; CHECK-NEXT: v_mov_b32_e32 v1, 0
-; CHECK-NEXT: s_lshl2_add_u32 s4, s17, 8
; CHECK-NEXT: v_mov_b32_e32 v2, 2
; CHECK-NEXT: v_mov_b32_e32 v3, s4
; CHECK-NEXT: ds_write_b16 v1, v0
diff --git a/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll b/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
index 3d79bdc25336d..6343cdaa0dff2 100644
--- a/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
+++ b/llvm/test/CodeGen/AMDGPU/lds-misaligned-bug.ll
@@ -168,9 +168,10 @@ define amdgpu_kernel void @test_local_misaligned_v3(ptr addrspace(3) %arg) {
; SPLIT-NEXT: v_lshl_add_u32 v2, v0, 2, s0
; SPLIT-NEXT: ds_read2_b32 v[0:1], v2 offset1:1
; SPLIT-NEXT: ds_read_b32 v3, v2 offset:8
-; SPLIT-NEXT: s_waitcnt lgkmcnt(0)
-; SPLIT-NEXT: ds_write2_b32 v2, v3, v0 offset1:1
+; SPLIT-NEXT: s_waitcnt lgkmcnt(1)
; SPLIT-NEXT: ds_write_b32 v2, v1 offset:8
+; SPLIT-NEXT: s_waitcnt lgkmcnt(1)
+; SPLIT-NEXT: ds_write2_b32 v2, v3, v0 offset1:1
; SPLIT-NEXT: s_endpgm
;
; ALIGNED-GFX10-LABEL: test_local_misaligned_v3:
@@ -180,9 +181,10 @@ define amdgpu_kernel void @test_local_misaligned_v3(ptr addrspace(3) %arg) {
; ALIGNED-GFX10-NEXT: v_lshl_add_u32 v2, v0, 2, s0
; ALIGNED-GFX10-NEXT: ds_read2_b32 v[0:1], v2 offset1:1
; ALIGNED-GFX10-NEXT: ds_read_b32 v3, v2 offset:8
-; ALIGNED-GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; ALIGNED-GFX10-NEXT: ds_write2_b32 v2, v3, v0 offset1:1
+; ALIGNED-GFX10-NEXT: s_waitcnt lgkmcnt(1)
; ALIGNED-GFX10-NEXT: ds_write_b32 v2, v1 offset:8
+; ALIGNED-GFX10-NEXT: s_waitcnt lgkmcnt(1)
+; ALIGNED-GFX10-NEXT: ds_write2_b32 v2, v3, v0 offset1:1
; ALIGNED-GFX10-NEXT: s_endpgm
;
; UNALIGNED-GFX10-LABEL: test_local_misaligned_v3:
@@ -192,9 +194,10 @@ define amdgpu_kernel void @test_local_misaligned_v3(ptr addrspace(3) %arg) {
; UNALIGNED-GFX10-NEXT: v_lshl_add_u32 v2, v0, 2, s0
; UNALIGNED-GFX10-NEXT: ds_read2_b32 v[0:1], v2 offset1:1
; UNALIGNED-GFX10-NEXT: ds_read_b32 v3, v2 offset:8
-; UNALIGNED-GFX10-NEXT: s_waitcnt lgkmcnt(0)
-; UNALIGNED-GFX10-NEXT: ds_write2_b32 v2, v3, v0 offset1:1
+; UNALIGNED-GFX10-NEXT: s_waitcnt lgkmcnt(1)
; UNALIGNED-GFX10-NEXT: ds_write_b32 v2, v1 offset:8
+; UNALIGNED-GFX10-NEXT: s_waitcnt lgkmcnt(1)
+; UNALIGNED-GFX10-NEXT: ds_write2_b32 v2, v3, v0 offset1:1
; UNALIGNED-GFX10-NEXT: s_endpgm
;
; ALIGNED-GFX11-LABEL: test_local_misaligned_v3:
@@ -206,9 +209,10 @@ define amdgpu_kernel void @test_local_misaligned_v3(ptr addrspace(3) %arg) {
; ALIGNED-GFX11-NEXT: v_lshl_add_u32 v2, v0, 2, s0
; ALIGNED-GFX11-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:1
; ALIGNED-GFX11-NEXT: ds_load_b32 v3, v2 offset:8
-; ALIGNED-GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; ALIGNED-GFX11-NEXT: ds_store_2addr_b32 v2, v3, v0 offset1:1
+; ALIGNED-GFX11-NEXT: s_waitcnt lgkmcnt(1)
; ALIGNED-GFX11-NEXT: ds_store_b32 v2, v1 offset:8
+; ALIGNED-GFX11-NEXT: s_waitcnt lgkmcnt(1)
+; ALIGNED-GFX11-NEXT: ds_store_2addr_b32 v2, v3, v0 offset1:1
; ALIGNED-GFX11-NEXT: s_endpgm
;
; UNALIGNED-GFX11-LABEL: test_local_misaligned_v3:
@@ -220,9 +224,10 @@ define amdgpu_kernel void @test_local_misaligned_v3(ptr addrspace(3) %arg) {
; UNALIGNED-GFX11-NEXT: v_lshl_add_u32 v2, v0, 2, s0
; UNALIGNED-GFX11-NEXT: ds_load_2addr_b32 v[0:1], v2 offset1:1
; UNALIGNED-GFX11-NEXT: ds_load_b32 v3, v2 offset:8
-; UNALIGNED-GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; UNALIGNED-GFX11-NEXT: ds_store_2addr_b32 v2, v3, v0 offset1:1
+; UNALIGNED-GFX11-NEXT: s_waitcnt lgkmcnt(1)
; UNALIGNED-GFX11-NEXT: ds_store_b32 v2, v1 offset:8
+; UNALIGNED-GFX11-NEXT: s_waitcnt lgkmcnt(1)
+; UNALIGNED-GFX11-NEXT: ds_store_2addr_b32 v2, v3, v0 offset1:1
; UNALIGNED-GFX11-NEXT: s_endpgm
bb:
%lid = tail call i32 @llvm.amdgcn.workitem.id.x()
diff --git a/llvm/test/CodeGen/AMDGPU/load-lo16.ll b/llvm/test/CodeGen/AMDGPU/load-lo16.ll
index 5e5c3bcd37f01..7358b1175a9dd 100644
--- a/llvm/test/CodeGen/AMDGPU/load-lo16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-lo16.ll
@@ -707,9 +707,9 @@ define void @load_local_lo_v2i16_reghi_vreg_multi_use_lohi(ptr addrspace(3) noal
; GFX900-MUBUF-NEXT: ds_read_u16 v0, v0
; GFX900-MUBUF-NEXT: s_mov_b32 s4, 0xffff
; GFX900-MUBUF-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX900-MUBUF-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-MUBUF-NEXT: ds_write_b16 v2, v0
; GFX900-MUBUF-NEXT: ds_write_b16 v3, v4
+; GFX900-MUBUF-NEXT: s_waitcnt lgkmcnt(1)
+; GFX900-MUBUF-NEXT: ds_write_b16 v2, v0
; GFX900-MUBUF-NEXT: v_bfi_b32 v0, s4, v0, v1
; GFX900-MUBUF-NEXT: global_store_dword v[0:1], v0, off
; GFX900-MUBUF-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -721,9 +721,9 @@ define void @load_local_lo_v2i16_reghi_vreg_multi_use_lohi(ptr addrspace(3) noal
; GFX906-NEXT: ds_read_u16 v0, v0
; GFX906-NEXT: s_mov_b32 s4, 0xffff
; GFX906-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX906-NEXT: s_waitcnt lgkmcnt(0)
-; GFX906-NEXT: ds_write_b16 v2, v0
; GFX906-NEXT: ds_write_b16 v3, v4
+; GFX906-NEXT: s_waitcnt lgkmcnt(1)
+; GFX906-NEXT: ds_write_b16 v2, v0
; GFX906-NEXT: v_bfi_b32 v0, s4, v0, v1
; GFX906-NEXT: global_store_dword v[0:1], v0, off
; GFX906-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -736,9 +736,9 @@ define void @load_local_lo_v2i16_reghi_vreg_multi_use_lohi(ptr addrspace(3) noal
; GFX803-NEXT: ds_read_u16 v0, v0
; GFX803-NEXT: s_mov_b32 s4, 0x3020504
; GFX803-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX803-NEXT: s_waitcnt lgkmcnt(0)
-; GFX803-NEXT: ds_write_b16 v2, v0
; GFX803-NEXT: ds_write_b16 v3, v4
+; GFX803-NEXT: s_waitcnt lgkmcnt(1)
+; GFX803-NEXT: ds_write_b16 v2, v0
; GFX803-NEXT: v_perm_b32 v0, v0, v1, s4
; GFX803-NEXT: flat_store_dword v[0:1], v0
; GFX803-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
@@ -750,9 +750,9 @@ define void @load_local_lo_v2i16_reghi_vreg_multi_use_lohi(ptr addrspace(3) noal
; GFX900-FLATSCR-NEXT: ds_read_u16 v0, v0
; GFX900-FLATSCR-NEXT: s_mov_b32 s0, 0xffff
; GFX900-FLATSCR-NEXT: v_lshrrev_b32_e32 v4, 16, v1
-; GFX900-FLATSCR-NEXT: s_waitcnt lgkmcnt(0)
-; GFX900-FLATSCR-NEXT: ds_write_b16 v2, v0
; GFX900-FLATSCR-NEXT: ds_write_b16 v3, v4
+; GFX900-FLATSCR-NEXT: s_waitcnt lgkmcnt(1)
+; GFX900-FLATSCR-NEXT: ds_write_b16 v2, v0
; GFX900-FLATSCR-NEXT: v_bfi_b32 v0, s0, v0, v1
; GFX900-FLATSCR-NEXT: global_store_dword v[0:1], v0, off
; GFX900-FLATSCR-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
index 04d906ca6ad9c..288a8336e09b6 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-i16.ll
@@ -1312,6 +1312,8 @@ define amdgpu_kernel void @local_zextload_v8i16_to_v8i32(ptr addrspace(3) %out,
; VI-DS128-NEXT: v_mov_b32_e32 v0, s1
; VI-DS128-NEXT: ds_read_b128 v[0:3], v0
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v1
+; VI-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v1
; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; VI-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v0
; VI-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v3
@@ -1319,8 +1321,6 @@ define amdgpu_kernel void @local_zextload_v8i16_to_v8i32(ptr addrspace(3) %out,
; VI-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v2
; VI-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v2
; VI-DS128-NEXT: v_mov_b32_e32 v0, s0
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v1
-; VI-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v1
; VI-DS128-NEXT: ds_write_b128 v0, v[8:11] offset:16
; VI-DS128-NEXT: ds_write_b128 v0, v[4:7]
; VI-DS128-NEXT: s_endpgm
@@ -1333,14 +1333,14 @@ define amdgpu_kernel void @local_zextload_v8i16_to_v8i32(ptr addrspace(3) %out,
; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v0
; GFX9-DS128-NEXT: v_mov_b32_e32 v12, s0
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v3
-; GFX9-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v3
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; GFX9-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v2
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v1
; GFX9-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v1
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v0
; GFX9-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v0
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v3
+; GFX9-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v3
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v2
+; GFX9-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v2
; GFX9-DS128-NEXT: ds_write_b128 v12, v[8:11] offset:16
; GFX9-DS128-NEXT: ds_write_b128 v12, v[4:7]
; GFX9-DS128-NEXT: s_endpgm
@@ -1381,14 +1381,14 @@ define amdgpu_kernel void @local_sextload_v8i16_to_v8i32(ptr addrspace(3) %out,
; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v0 offset1:1
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v0
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v1
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v2
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v3
; VI-NO-DS128-NEXT: v_bfe_i32 v4, v0, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v6, v1, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v8, v2, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v10, v3, 0, 16
; VI-NO-DS128-NEXT: v_mov_b32_e32 v0, s0
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v1
-; VI-NO-DS128-NEXT: v_bfe_i32 v6, v1, 0, 16
; VI-NO-DS128-NEXT: ds_write2_b64 v0, v[8:9], v[10:11] offset0:2 offset1:3
; VI-NO-DS128-NEXT: ds_write2_b64 v0, v[4:5], v[6:7] offset1:1
; VI-NO-DS128-NEXT: s_endpgm
@@ -1401,14 +1401,14 @@ define amdgpu_kernel void @local_sextload_v8i16_to_v8i32(ptr addrspace(3) %out,
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v0 offset1:1
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v12, s0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v2
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v3
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v8, v2, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v10, v3, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v0
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v1
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v2
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v3
; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v0, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v1, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v8, v2, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v10, v3, 0, 16
; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[8:9], v[10:11] offset0:2 offset1:3
; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[4:5], v[6:7] offset1:1
; GFX9-NO-DS128-NEXT: s_endpgm
@@ -1478,15 +1478,15 @@ define amdgpu_kernel void @local_sextload_v8i16_to_v8i32(ptr addrspace(3) %out,
; VI-DS128-NEXT: v_mov_b32_e32 v0, s1
; VI-DS128-NEXT: ds_read_b128 v[0:3], v0
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v1
; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v0
; VI-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v3
; VI-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v2
+; VI-DS128-NEXT: v_bfe_i32 v6, v1, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v4, v0, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v10, v3, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v8, v2, 0, 16
; VI-DS128-NEXT: v_mov_b32_e32 v0, s0
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v1
-; VI-DS128-NEXT: v_bfe_i32 v6, v1, 0, 16
; VI-DS128-NEXT: ds_write_b128 v0, v[8:11] offset:16
; VI-DS128-NEXT: ds_write_b128 v0, v[4:7]
; VI-DS128-NEXT: s_endpgm
@@ -1499,14 +1499,14 @@ define amdgpu_kernel void @local_sextload_v8i16_to_v8i32(ptr addrspace(3) %out,
; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v0
; GFX9-DS128-NEXT: v_mov_b32_e32 v12, s0
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v3
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v2
-; GFX9-DS128-NEXT: v_bfe_i32 v10, v3, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v8, v2, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v1
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v3
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v2
; GFX9-DS128-NEXT: v_bfe_i32 v6, v1, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v4, v0, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v10, v3, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v8, v2, 0, 16
; GFX9-DS128-NEXT: ds_write_b128 v12, v[8:11] offset:16
; GFX9-DS128-NEXT: ds_write_b128 v12, v[4:7]
; GFX9-DS128-NEXT: s_endpgm
@@ -1561,11 +1561,6 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: v_mov_b32_e32 v16, s0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v7
-; VI-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v7
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
; VI-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v1
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -1573,10 +1568,15 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v3
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; VI-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v5
; VI-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v5
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; VI-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v7
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
; VI-NO-DS128-NEXT: ds_write2_b64 v16, v[6:7], v[14:15] offset0:6 offset1:7
; VI-NO-DS128-NEXT: ds_write2_b64 v16, v[4:5], v[12:13] offset0:4 offset1:5
; VI-NO-DS128-NEXT: ds_write2_b64 v16, v[2:3], v[10:11] offset0:2 offset1:3
@@ -1593,11 +1593,6 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i32(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v16, s0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v1
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v7
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v7
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v1
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -1605,10 +1600,15 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i32(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v3
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v5
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v5
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v7
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v7
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[6:7], v[14:15] offset0:6 offset1:7
; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[4:5], v[12:13] offset0:4 offset1:5
; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[2:3], v[10:11] offset0:2 offset1:3
@@ -1728,7 +1728,13 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i32(ptr addrspace(3) %out
; VI-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v1
; VI-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v0
; VI-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v3
+; VI-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v3
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v2
+; VI-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v2
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; VI-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v5
; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; VI-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v4
; VI-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v7
@@ -1736,12 +1742,6 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i32(ptr addrspace(3) %out
; VI-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v6
; VI-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v6
; VI-DS128-NEXT: v_mov_b32_e32 v4, s0
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v3
-; VI-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v3
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v2
-; VI-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v2
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; VI-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v5
; VI-DS128-NEXT: ds_write_b128 v4, v[16:19] offset:48
; VI-DS128-NEXT: ds_write_b128 v4, v[0:3] offset:32
; VI-DS128-NEXT: ds_write_b128 v4, v[12:15] offset:16
@@ -1760,7 +1760,13 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i32(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v1
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v0
; GFX9-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v3
+; GFX9-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v3
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v2
+; GFX9-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v2
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v5
+; GFX9-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v5
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v4
; GFX9-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v4
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v7
@@ -1768,12 +1774,6 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i32(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v6
; GFX9-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v6
; GFX9-DS128-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v3
-; GFX9-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v3
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v2
-; GFX9-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v2
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v5
-; GFX9-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v5
; GFX9-DS128-NEXT: ds_write_b128 v4, v[16:19] offset:48
; GFX9-DS128-NEXT: ds_write_b128 v4, v[0:3] offset:32
; GFX9-DS128-NEXT: ds_write_b128 v4, v[12:15] offset:16
@@ -1833,6 +1833,7 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 16, v3
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v2
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v5
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v4
; VI-NO-DS128-NEXT: v_bfe_i32 v8, v1, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v10, v0, 0, 16
@@ -1840,12 +1841,11 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: v_bfe_i32 v14, v2, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v7
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v6
+; VI-NO-DS128-NEXT: v_bfe_i32 v16, v5, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v18, v4, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v0, v7, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v2, v6, 0, 16
; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, s0
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v5
-; VI-NO-DS128-NEXT: v_bfe_i32 v16, v5, 0, 16
; VI-NO-DS128-NEXT: ds_write2_b64 v4, v[2:3], v[0:1] offset0:6 offset1:7
; VI-NO-DS128-NEXT: ds_write2_b64 v4, v[18:19], v[16:17] offset0:4 offset1:5
; VI-NO-DS128-NEXT: ds_write2_b64 v4, v[14:15], v[12:13] offset0:2 offset1:3
@@ -1865,6 +1865,7 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i32(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 16, v3
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v2
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v5
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v4
; GFX9-NO-DS128-NEXT: v_bfe_i32 v8, v1, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v10, v0, 0, 16
@@ -1872,12 +1873,11 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i32(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_bfe_i32 v14, v2, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v7
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v6
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v5, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v4, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v7, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v6, 0, 16
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v5
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v5, 0, 16
; GFX9-NO-DS128-NEXT: ds_write2_b64 v4, v[2:3], v[0:1] offset0:6 offset1:7
; GFX9-NO-DS128-NEXT: ds_write2_b64 v4, v[18:19], v[16:17] offset0:4 offset1:5
; GFX9-NO-DS128-NEXT: ds_write2_b64 v4, v[14:15], v[12:13] offset0:2 offset1:3
@@ -2008,6 +2008,7 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i32(ptr addrspace(3) %out
; VI-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v3
; VI-DS128-NEXT: v_ashrrev_i32_e32 v13, 16, v2
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v5
; VI-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v4
; VI-DS128-NEXT: v_bfe_i32 v10, v1, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v8, v0, 0, 16
@@ -2015,12 +2016,11 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i32(ptr addrspace(3) %out
; VI-DS128-NEXT: v_bfe_i32 v12, v2, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v7
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v6
+; VI-DS128-NEXT: v_bfe_i32 v18, v5, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v2, v7, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v0, v6, 0, 16
; VI-DS128-NEXT: v_mov_b32_e32 v4, s0
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v5
-; VI-DS128-NEXT: v_bfe_i32 v18, v5, 0, 16
; VI-DS128-NEXT: ds_write_b128 v4, v[0:3] offset:48
; VI-DS128-NEXT: ds_write_b128 v4, v[16:19] offset:32
; VI-DS128-NEXT: ds_write_b128 v4, v[12:15] offset:16
@@ -2040,6 +2040,7 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i32(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v3
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 16, v2
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v5
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v4
; GFX9-DS128-NEXT: v_bfe_i32 v10, v1, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v8, v0, 0, 16
@@ -2047,12 +2048,11 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i32(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_bfe_i32 v12, v2, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v7
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v6
+; GFX9-DS128-NEXT: v_bfe_i32 v18, v5, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v2, v7, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v0, v6, 0, 16
; GFX9-DS128-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v5
-; GFX9-DS128-NEXT: v_bfe_i32 v18, v5, 0, 16
; GFX9-DS128-NEXT: ds_write_b128 v4, v[0:3] offset:48
; GFX9-DS128-NEXT: ds_write_b128 v4, v[16:19] offset:32
; GFX9-DS128-NEXT: ds_write_b128 v4, v[12:15] offset:16
@@ -2127,9 +2127,9 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NO-DS128-NEXT: s_mov_b32 m0, -1
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v24, s1
-; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v24 offset1:1
-; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v24 offset0:2 offset1:3
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v26, s1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v26 offset1:1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v26 offset0:2 offset1:3
; VI-NO-DS128-NEXT: v_mov_b32_e32 v32, s0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v3
@@ -2145,30 +2145,30 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v7
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v6
; VI-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v6
+; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v26 offset0:4 offset1:5
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v5
-; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v24 offset0:4 offset1:5
; VI-NO-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v5
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v4
; VI-NO-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v4
-; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v24 offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v26 offset0:6 offset1:7
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v1
; VI-NO-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v1
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v5
-; VI-NO-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v5
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v27, 16, v3
; VI-NO-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v3
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; VI-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v7
; VI-NO-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v7
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v5
+; VI-NO-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v5
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[4:5], v[30:31] offset0:12 offset1:13
; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[6:7], v[28:29] offset0:14 offset1:15
; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[2:3], v[26:27] offset0:10 offset1:11
@@ -2183,9 +2183,9 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i32(ptr addrspace(3) %out
; GFX9-NO-DS128: ; %bb.0:
; GFX9-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v24, s1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v24 offset1:1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v24 offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v26, s1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v26 offset1:1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v26 offset0:2 offset1:3
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v32, s0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v3
@@ -2201,30 +2201,30 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i32(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v7
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v6
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v6
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v26 offset0:4 offset1:5
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v5
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v24 offset0:4 offset1:5
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v5
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v4
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v4
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v24 offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v26 offset0:6 offset1:7
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v1
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v1
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v5
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v5
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v27, 16, v3
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v3
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v7
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v7
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v5
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v5
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[4:5], v[30:31] offset0:12 offset1:13
; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[6:7], v[28:29] offset0:14 offset1:15
; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[2:3], v[26:27] offset0:10 offset1:11
@@ -2437,30 +2437,21 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i32(ptr addrspace(3) %out
; VI-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-DS128-NEXT: s_mov_b32 m0, -1
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_mov_b32_e32 v20, s1
-; VI-DS128-NEXT: ds_read_b128 v[0:3], v20
-; VI-DS128-NEXT: ds_read_b128 v[4:7], v20 offset:16
-; VI-DS128-NEXT: ds_read_b128 v[16:19], v20 offset:32
-; VI-DS128-NEXT: ds_read_b128 v[20:23], v20 offset:48
-; VI-DS128-NEXT: v_mov_b32_e32 v32, s0
+; VI-DS128-NEXT: v_mov_b32_e32 v22, s1
+; VI-DS128-NEXT: ds_read_b128 v[0:3], v22
+; VI-DS128-NEXT: ds_read_b128 v[4:7], v22 offset:16
+; VI-DS128-NEXT: ds_read_b128 v[16:19], v22 offset:32
+; VI-DS128-NEXT: ds_read_b128 v[22:25], v22 offset:48
; VI-DS128-NEXT: s_waitcnt lgkmcnt(3)
; VI-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v3
; VI-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v3
; VI-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v23
-; VI-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v23
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v22
-; VI-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v22
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v21
-; VI-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v21
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v20
-; VI-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v20
; VI-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v2
; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v1
; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; VI-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(2)
; VI-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v7
; VI-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v7
; VI-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v6
@@ -2469,52 +2460,54 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i32(ptr addrspace(3) %out
; VI-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v5
; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; VI-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v27, 16, v19
-; VI-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v19
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v18
-; VI-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v18
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v17
-; VI-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v17
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v16
-; VI-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; VI-DS128-NEXT: ds_write_b128 v32, v[20:23] offset:96
-; VI-DS128-NEXT: ds_write_b128 v32, v[28:31] offset:112
-; VI-DS128-NEXT: ds_write_b128 v32, v[16:19] offset:64
-; VI-DS128-NEXT: ds_write_b128 v32, v[24:27] offset:80
-; VI-DS128-NEXT: ds_write_b128 v32, v[4:7] offset:32
-; VI-DS128-NEXT: ds_write_b128 v32, v[12:15] offset:48
-; VI-DS128-NEXT: ds_write_b128 v32, v[0:3]
-; VI-DS128-NEXT: ds_write_b128 v32, v[8:11] offset:16
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v19
+; VI-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v19
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v18
+; VI-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v17
+; VI-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v17
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v27, 16, v16
+; VI-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v16
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v33, 16, v25
+; VI-DS128-NEXT: v_and_b32_e32 v32, 0xffff, v25
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v24
+; VI-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v24
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v23
+; VI-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v23
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v22
+; VI-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; VI-DS128-NEXT: v_mov_b32_e32 v16, s0
+; VI-DS128-NEXT: ds_write_b128 v16, v[22:25] offset:96
+; VI-DS128-NEXT: ds_write_b128 v16, v[30:33] offset:112
+; VI-DS128-NEXT: ds_write_b128 v16, v[26:29] offset:64
+; VI-DS128-NEXT: ds_write_b128 v16, v[18:21] offset:80
+; VI-DS128-NEXT: ds_write_b128 v16, v[4:7] offset:32
+; VI-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:48
+; VI-DS128-NEXT: ds_write_b128 v16, v[0:3]
+; VI-DS128-NEXT: ds_write_b128 v16, v[8:11] offset:16
; VI-DS128-NEXT: s_endpgm
;
; GFX9-DS128-LABEL: local_zextload_v32i16_to_v32i32:
; GFX9-DS128: ; %bb.0:
; GFX9-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_mov_b32_e32 v20, s1
-; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v20
-; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v20 offset:16
-; GFX9-DS128-NEXT: ds_read_b128 v[16:19], v20 offset:32
-; GFX9-DS128-NEXT: ds_read_b128 v[20:23], v20 offset:48
-; GFX9-DS128-NEXT: v_mov_b32_e32 v32, s0
+; GFX9-DS128-NEXT: v_mov_b32_e32 v22, s1
+; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v22
+; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v22 offset:16
+; GFX9-DS128-NEXT: ds_read_b128 v[16:19], v22 offset:32
+; GFX9-DS128-NEXT: ds_read_b128 v[22:25], v22 offset:48
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(3)
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v3
; GFX9-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v3
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v2
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v23
-; GFX9-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v23
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v22
-; GFX9-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v22
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v21
-; GFX9-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v21
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v20
-; GFX9-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v20
; GFX9-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v2
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v1
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX9-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(2)
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v7
; GFX9-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v7
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v6
@@ -2523,22 +2516,33 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i32(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v5
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v4
; GFX9-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v27, 16, v19
-; GFX9-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v19
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v18
-; GFX9-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v18
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v17
-; GFX9-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v17
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v16
-; GFX9-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[20:23] offset:96
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[28:31] offset:112
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[16:19] offset:64
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[24:27] offset:80
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[4:7] offset:32
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[12:15] offset:48
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[0:3]
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[8:11] offset:16
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v19
+; GFX9-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v19
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v18
+; GFX9-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v17
+; GFX9-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v17
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v27, 16, v16
+; GFX9-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v16
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v33, 16, v25
+; GFX9-DS128-NEXT: v_and_b32_e32 v32, 0xffff, v25
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v24
+; GFX9-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v24
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v23
+; GFX9-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v23
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v22
+; GFX9-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; GFX9-DS128-NEXT: v_mov_b32_e32 v16, s0
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[22:25] offset:96
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[30:33] offset:112
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[26:29] offset:64
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[18:21] offset:80
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[4:7] offset:32
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:48
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[0:3]
+; GFX9-DS128-NEXT: ds_write_b128 v16, v[8:11] offset:16
; GFX9-DS128-NEXT: s_endpgm
%load = load <32 x i16>, ptr addrspace(3) %in
%ext = zext <32 x i16> %load to <32 x i32>
@@ -2609,9 +2613,9 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NO-DS128-NEXT: s_mov_b32 m0, -1
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v24, s1
-; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v24 offset1:1
-; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v24 offset0:2 offset1:3
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v26, s1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v26 offset1:1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v26 offset0:2 offset1:3
; VI-NO-DS128-NEXT: v_mov_b32_e32 v32, s0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v3
@@ -2627,30 +2631,30 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v6
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v21, 16, v5
; VI-NO-DS128-NEXT: v_bfe_i32 v16, v7, 0, 16
+; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v26 offset0:4 offset1:5
; VI-NO-DS128-NEXT: v_bfe_i32 v18, v6, 0, 16
-; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v24 offset0:4 offset1:5
; VI-NO-DS128-NEXT: v_bfe_i32 v20, v5, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v23, 16, v4
; VI-NO-DS128-NEXT: v_bfe_i32 v22, v4, 0, 16
-; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v24 offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v26 offset0:6 offset1:7
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v25, 16, v1
; VI-NO-DS128-NEXT: v_bfe_i32 v24, v1, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v0
; VI-NO-DS128-NEXT: v_bfe_i32 v0, v0, 0, 16
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v31, 16, v5
-; VI-NO-DS128-NEXT: v_bfe_i32 v30, v5, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v4
-; VI-NO-DS128-NEXT: v_bfe_i32 v4, v4, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v27, 16, v3
; VI-NO-DS128-NEXT: v_bfe_i32 v26, v3, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v2
; VI-NO-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v29, 16, v7
; VI-NO-DS128-NEXT: v_bfe_i32 v28, v7, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v6
; VI-NO-DS128-NEXT: v_bfe_i32 v6, v6, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v31, 16, v5
+; VI-NO-DS128-NEXT: v_bfe_i32 v30, v5, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v4
+; VI-NO-DS128-NEXT: v_bfe_i32 v4, v4, 0, 16
; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[4:5], v[30:31] offset0:12 offset1:13
; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[6:7], v[28:29] offset0:14 offset1:15
; VI-NO-DS128-NEXT: ds_write2_b64 v32, v[2:3], v[26:27] offset0:10 offset1:11
@@ -2665,9 +2669,9 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; GFX9-NO-DS128: ; %bb.0:
; GFX9-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v24, s1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v24 offset1:1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v24 offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v26, s1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v26 offset1:1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v26 offset0:2 offset1:3
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v32, s0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v3
@@ -2683,30 +2687,30 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v6
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v21, 16, v5
; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v7, 0, 16
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v26 offset0:4 offset1:5
; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v6, 0, 16
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v24 offset0:4 offset1:5
; GFX9-NO-DS128-NEXT: v_bfe_i32 v20, v5, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v23, 16, v4
; GFX9-NO-DS128-NEXT: v_bfe_i32 v22, v4, 0, 16
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v24 offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v26 offset0:6 offset1:7
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v25, 16, v1
; GFX9-NO-DS128-NEXT: v_bfe_i32 v24, v1, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v0
; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v0, 0, 16
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v31, 16, v5
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v30, v5, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v4
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v4, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v27, 16, v3
; GFX9-NO-DS128-NEXT: v_bfe_i32 v26, v3, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v2
; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v29, 16, v7
; GFX9-NO-DS128-NEXT: v_bfe_i32 v28, v7, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v6
; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v6, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v31, 16, v5
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v30, v5, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v4
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v4, 0, 16
; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[4:5], v[30:31] offset0:12 offset1:13
; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[6:7], v[28:29] offset0:14 offset1:15
; GFX9-NO-DS128-NEXT: ds_write2_b64 v32, v[2:3], v[26:27] offset0:10 offset1:11
@@ -2938,11 +2942,12 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; VI-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-DS128-NEXT: s_mov_b32 m0, -1
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_mov_b32_e32 v24, s1
-; VI-DS128-NEXT: ds_read_b128 v[0:3], v24
-; VI-DS128-NEXT: ds_read_b128 v[4:7], v24 offset:16
-; VI-DS128-NEXT: ds_read_b128 v[20:23], v24 offset:32
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(2)
+; VI-DS128-NEXT: v_mov_b32_e32 v25, s1
+; VI-DS128-NEXT: ds_read_b128 v[0:3], v25
+; VI-DS128-NEXT: ds_read_b128 v[4:7], v25 offset:16
+; VI-DS128-NEXT: ds_read_b128 v[20:23], v25 offset:32
+; VI-DS128-NEXT: ds_read_b128 v[27:30], v25 offset:48
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(3)
; VI-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v3
; VI-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v2
; VI-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v1
@@ -2951,7 +2956,7 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; VI-DS128-NEXT: v_bfe_i32 v8, v2, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v14, v1, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v12, v0, 0, 16
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(2)
; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v7
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v6
; VI-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v5
@@ -2960,45 +2965,45 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; VI-DS128-NEXT: v_bfe_i32 v0, v6, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v18, v5, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
-; VI-DS128-NEXT: ds_read_b128 v[4:7], v24 offset:48
; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v26, 16, v23
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v24, 16, v22
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v30, 16, v21
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v28, 16, v20
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v23
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v22
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v26, 16, v21
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v24, 16, v20
+; VI-DS128-NEXT: v_bfe_i32 v6, v23, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v4, v22, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v25, v21, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v23, v20, 0, 16
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v38, 16, v5
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v36, 16, v4
-; VI-DS128-NEXT: v_bfe_i32 v37, v5, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v35, v4, 0, 16
-; VI-DS128-NEXT: v_mov_b32_e32 v4, s0
-; VI-DS128-NEXT: v_bfe_i32 v25, v23, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v23, v22, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v29, v21, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v27, v20, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v34, 16, v7
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v32, 16, v6
-; VI-DS128-NEXT: v_bfe_i32 v33, v7, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v31, v6, 0, 16
-; VI-DS128-NEXT: ds_write_b128 v4, v[35:38] offset:96
-; VI-DS128-NEXT: ds_write_b128 v4, v[31:34] offset:112
-; VI-DS128-NEXT: ds_write_b128 v4, v[27:30] offset:64
-; VI-DS128-NEXT: ds_write_b128 v4, v[23:26] offset:80
-; VI-DS128-NEXT: ds_write_b128 v4, v[16:19] offset:32
-; VI-DS128-NEXT: ds_write_b128 v4, v[0:3] offset:48
-; VI-DS128-NEXT: ds_write_b128 v4, v[12:15]
-; VI-DS128-NEXT: ds_write_b128 v4, v[8:11] offset:16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v33, 16, v30
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v31, 16, v29
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v37, 16, v28
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v35, 16, v27
+; VI-DS128-NEXT: v_bfe_i32 v32, v30, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v30, v29, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v36, v28, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v34, v27, 0, 16
+; VI-DS128-NEXT: v_mov_b32_e32 v20, s0
+; VI-DS128-NEXT: ds_write_b128 v20, v[34:37] offset:96
+; VI-DS128-NEXT: ds_write_b128 v20, v[30:33] offset:112
+; VI-DS128-NEXT: ds_write_b128 v20, v[23:26] offset:64
+; VI-DS128-NEXT: ds_write_b128 v20, v[4:7] offset:80
+; VI-DS128-NEXT: ds_write_b128 v20, v[16:19] offset:32
+; VI-DS128-NEXT: ds_write_b128 v20, v[0:3] offset:48
+; VI-DS128-NEXT: ds_write_b128 v20, v[12:15]
+; VI-DS128-NEXT: ds_write_b128 v20, v[8:11] offset:16
; VI-DS128-NEXT: s_endpgm
;
; GFX9-DS128-LABEL: local_sextload_v32i16_to_v32i32:
; GFX9-DS128: ; %bb.0:
; GFX9-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_mov_b32_e32 v24, s1
-; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v24
-; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v24 offset:16
-; GFX9-DS128-NEXT: ds_read_b128 v[20:23], v24 offset:32
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(2)
+; GFX9-DS128-NEXT: v_mov_b32_e32 v25, s1
+; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v25
+; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v25 offset:16
+; GFX9-DS128-NEXT: ds_read_b128 v[20:23], v25 offset:32
+; GFX9-DS128-NEXT: ds_read_b128 v[27:30], v25 offset:48
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(3)
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v3
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v2
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v1
@@ -3007,7 +3012,7 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_bfe_i32 v8, v2, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v14, v1, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v12, v0, 0, 16
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(2)
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v7
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v6
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v5
@@ -3016,34 +3021,33 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i32(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_bfe_i32 v0, v6, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v18, v5, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
-; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v24 offset:48
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v26, 16, v23
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v24, 16, v22
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v30, 16, v21
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v28, 16, v20
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v23
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v22
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v26, 16, v21
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v24, 16, v20
+; GFX9-DS128-NEXT: v_bfe_i32 v6, v23, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v4, v22, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v25, v21, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v23, v20, 0, 16
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v38, 16, v5
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v36, 16, v4
-; GFX9-DS128-NEXT: v_bfe_i32 v37, v5, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v35, v4, 0, 16
-; GFX9-DS128-NEXT: v_mov_b32_e32 v4, s0
-; GFX9-DS128-NEXT: v_bfe_i32 v25, v23, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v23, v22, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v29, v21, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v27, v20, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v34, 16, v7
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v32, 16, v6
-; GFX9-DS128-NEXT: v_bfe_i32 v33, v7, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v31, v6, 0, 16
-; GFX9-DS128-NEXT: ds_write_b128 v4, v[35:38] offset:96
-; GFX9-DS128-NEXT: ds_write_b128 v4, v[31:34] offset:112
-; GFX9-DS128-NEXT: ds_write_b128 v4, v[27:30] offset:64
-; GFX9-DS128-NEXT: ds_write_b128 v4, v[23:26] offset:80
-; GFX9-DS128-NEXT: ds_write_b128 v4, v[16:19] offset:32
-; GFX9-DS128-NEXT: ds_write_b128 v4, v[0:3] offset:48
-; GFX9-DS128-NEXT: ds_write_b128 v4, v[12:15]
-; GFX9-DS128-NEXT: ds_write_b128 v4, v[8:11] offset:16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v33, 16, v30
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v31, 16, v29
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v37, 16, v28
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v35, 16, v27
+; GFX9-DS128-NEXT: v_bfe_i32 v32, v30, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v30, v29, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v36, v28, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v34, v27, 0, 16
+; GFX9-DS128-NEXT: v_mov_b32_e32 v20, s0
+; GFX9-DS128-NEXT: ds_write_b128 v20, v[34:37] offset:96
+; GFX9-DS128-NEXT: ds_write_b128 v20, v[30:33] offset:112
+; GFX9-DS128-NEXT: ds_write_b128 v20, v[23:26] offset:64
+; GFX9-DS128-NEXT: ds_write_b128 v20, v[4:7] offset:80
+; GFX9-DS128-NEXT: ds_write_b128 v20, v[16:19] offset:32
+; GFX9-DS128-NEXT: ds_write_b128 v20, v[0:3] offset:48
+; GFX9-DS128-NEXT: ds_write_b128 v20, v[12:15]
+; GFX9-DS128-NEXT: ds_write_b128 v20, v[8:11] offset:16
; GFX9-DS128-NEXT: s_endpgm
%load = load <32 x i16>, ptr addrspace(3) %in
%ext = sext <32 x i16> %load to <32 x i32>
@@ -3062,73 +3066,73 @@ define amdgpu_kernel void @local_zextload_v64i16_to_v64i32(ptr addrspace(3) %out
; SI-NEXT: s_addc_u32 s13, s13, 0
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
-; SI-NEXT: v_mov_b32_e32 v24, s1
+; SI-NEXT: v_mov_b32_e32 v20, s1
; SI-NEXT: s_mov_b32 m0, -1
-; SI-NEXT: ds_read2_b64 v[0:3], v24 offset0:8 offset1:9
-; SI-NEXT: ds_read2_b64 v[4:7], v24 offset0:10 offset1:11
-; SI-NEXT: ds_read2_b64 v[12:15], v24 offset0:12 offset1:13
-; SI-NEXT: ds_read2_b64 v[8:11], v24 offset0:14 offset1:15
-; SI-NEXT: ds_read2_b64 v[20:23], v24 offset1:1
-; SI-NEXT: ds_read2_b64 v[16:19], v24 offset0:2 offset1:3
-; SI-NEXT: ds_read2_b64 v[34:37], v24 offset0:4 offset1:5
-; SI-NEXT: ds_read2_b64 v[38:41], v24 offset0:6 offset1:7
+; SI-NEXT: ds_read2_b64 v[0:3], v20 offset0:8 offset1:9
+; SI-NEXT: ds_read2_b64 v[4:7], v20 offset0:10 offset1:11
+; SI-NEXT: ds_read2_b64 v[8:11], v20 offset0:12 offset1:13
+; SI-NEXT: ds_read2_b64 v[12:15], v20 offset0:14 offset1:15
+; SI-NEXT: ds_read2_b64 v[16:19], v20 offset1:1
+; SI-NEXT: ds_read2_b64 v[30:33], v20 offset0:2 offset1:3
+; SI-NEXT: ds_read2_b64 v[34:37], v20 offset0:4 offset1:5
+; SI-NEXT: ds_read2_b64 v[38:41], v20 offset0:6 offset1:7
; SI-NEXT: s_waitcnt lgkmcnt(7)
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v1
-; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v0
-; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v3
-; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v2
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v1
+; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v0
+; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v3
+; SI-NEXT: v_lshrrev_b32_e32 v27, 16, v2
; SI-NEXT: s_waitcnt lgkmcnt(6)
-; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v5
-; SI-NEXT: v_and_b32_e32 v24, 0xffff, v1
-; SI-NEXT: buffer_store_dword v24, off, s[12:15], 0 ; 4-byte Folded Spill
-; SI-NEXT: buffer_store_dword v25, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
-; SI-NEXT: v_and_b32_e32 v26, 0xffff, v0
-; SI-NEXT: v_and_b32_e32 v28, 0xffff, v3
-; SI-NEXT: v_and_b32_e32 v30, 0xffff, v2
+; SI-NEXT: v_lshrrev_b32_e32 v29, 16, v5
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v1
+; SI-NEXT: buffer_store_dword v20, off, s[12:15], 0 ; 4-byte Folded Spill
+; SI-NEXT: buffer_store_dword v21, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
+; SI-NEXT: v_and_b32_e32 v22, 0xffff, v0
+; SI-NEXT: v_and_b32_e32 v24, 0xffff, v3
+; SI-NEXT: v_and_b32_e32 v26, 0xffff, v2
; SI-NEXT: s_waitcnt expcnt(0)
-; SI-NEXT: v_lshrrev_b32_e32 v25, 16, v4
+; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v4
; SI-NEXT: v_lshrrev_b32_e32 v3, 16, v7
-; SI-NEXT: v_and_b32_e32 v32, 0xffff, v5
-; SI-NEXT: v_and_b32_e32 v24, 0xffff, v4
+; SI-NEXT: v_and_b32_e32 v28, 0xffff, v5
+; SI-NEXT: v_and_b32_e32 v20, 0xffff, v4
; SI-NEXT: v_and_b32_e32 v2, 0xffff, v7
; SI-NEXT: v_lshrrev_b32_e32 v5, 16, v6
; SI-NEXT: v_and_b32_e32 v4, 0xffff, v6
; SI-NEXT: s_waitcnt lgkmcnt(5)
-; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v13
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v13
-; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
-; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
-; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v15
-; SI-NEXT: v_and_b32_e32 v42, 0xffff, v15
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
-; SI-NEXT: s_waitcnt lgkmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v9
-; SI-NEXT: v_and_b32_e32 v44, 0xffff, v9
+; SI-NEXT: v_lshrrev_b32_e32 v7, 16, v9
+; SI-NEXT: v_and_b32_e32 v6, 0xffff, v9
; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v8
; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v11
-; SI-NEXT: v_and_b32_e32 v46, 0xffff, v11
+; SI-NEXT: v_lshrrev_b32_e32 v43, 16, v11
+; SI-NEXT: v_and_b32_e32 v42, 0xffff, v11
; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v10
; SI-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; SI-NEXT: s_waitcnt lgkmcnt(4)
+; SI-NEXT: v_lshrrev_b32_e32 v45, 16, v13
+; SI-NEXT: v_and_b32_e32 v44, 0xffff, v13
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v12
+; SI-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; SI-NEXT: v_lshrrev_b32_e32 v47, 16, v15
+; SI-NEXT: v_and_b32_e32 v46, 0xffff, v15
+; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
+; SI-NEXT: v_and_b32_e32 v14, 0xffff, v14
; SI-NEXT: s_waitcnt lgkmcnt(3)
-; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v21
-; SI-NEXT: v_and_b32_e32 v48, 0xffff, v21
-; SI-NEXT: v_lshrrev_b32_e32 v21, 16, v20
-; SI-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v23
-; SI-NEXT: v_and_b32_e32 v50, 0xffff, v23
-; SI-NEXT: v_lshrrev_b32_e32 v23, 16, v22
-; SI-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; SI-NEXT: s_waitcnt lgkmcnt(2)
-; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v17
-; SI-NEXT: v_and_b32_e32 v52, 0xffff, v17
+; SI-NEXT: v_lshrrev_b32_e32 v49, 16, v17
+; SI-NEXT: v_and_b32_e32 v48, 0xffff, v17
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v16
; SI-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v19
-; SI-NEXT: v_and_b32_e32 v54, 0xffff, v19
+; SI-NEXT: v_lshrrev_b32_e32 v51, 16, v19
+; SI-NEXT: v_and_b32_e32 v50, 0xffff, v19
; SI-NEXT: v_lshrrev_b32_e32 v19, 16, v18
; SI-NEXT: v_and_b32_e32 v18, 0xffff, v18
+; SI-NEXT: s_waitcnt lgkmcnt(2)
+; SI-NEXT: v_lshrrev_b32_e32 v53, 16, v31
+; SI-NEXT: v_and_b32_e32 v52, 0xffff, v31
+; SI-NEXT: v_lshrrev_b32_e32 v31, 16, v30
+; SI-NEXT: v_and_b32_e32 v30, 0xffff, v30
+; SI-NEXT: v_lshrrev_b32_e32 v55, 16, v33
+; SI-NEXT: v_and_b32_e32 v54, 0xffff, v33
+; SI-NEXT: v_lshrrev_b32_e32 v33, 16, v32
+; SI-NEXT: v_and_b32_e32 v32, 0xffff, v32
; SI-NEXT: s_waitcnt lgkmcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v57, 16, v35
; SI-NEXT: v_and_b32_e32 v56, 0xffff, v35
@@ -3152,21 +3156,21 @@ define amdgpu_kernel void @local_zextload_v64i16_to_v64i32(ptr addrspace(3) %out
; SI-NEXT: ds_write2_b64 v0, v[38:39], v[60:61] offset0:12 offset1:13
; SI-NEXT: ds_write2_b64 v0, v[36:37], v[58:59] offset0:10 offset1:11
; SI-NEXT: ds_write2_b64 v0, v[34:35], v[56:57] offset0:8 offset1:9
-; SI-NEXT: ds_write2_b64 v0, v[18:19], v[54:55] offset0:6 offset1:7
-; SI-NEXT: ds_write2_b64 v0, v[16:17], v[52:53] offset0:4 offset1:5
-; SI-NEXT: ds_write2_b64 v0, v[22:23], v[50:51] offset0:2 offset1:3
-; SI-NEXT: ds_write2_b64 v0, v[20:21], v[48:49] offset1:1
-; SI-NEXT: ds_write2_b64 v0, v[10:11], v[46:47] offset0:30 offset1:31
-; SI-NEXT: ds_write2_b64 v0, v[8:9], v[44:45] offset0:28 offset1:29
-; SI-NEXT: ds_write2_b64 v0, v[14:15], v[42:43] offset0:26 offset1:27
-; SI-NEXT: ds_write2_b64 v0, v[12:13], v[6:7] offset0:24 offset1:25
+; SI-NEXT: ds_write2_b64 v0, v[32:33], v[54:55] offset0:6 offset1:7
+; SI-NEXT: ds_write2_b64 v0, v[30:31], v[52:53] offset0:4 offset1:5
+; SI-NEXT: ds_write2_b64 v0, v[18:19], v[50:51] offset0:2 offset1:3
+; SI-NEXT: ds_write2_b64 v0, v[16:17], v[48:49] offset1:1
+; SI-NEXT: ds_write2_b64 v0, v[14:15], v[46:47] offset0:30 offset1:31
+; SI-NEXT: ds_write2_b64 v0, v[12:13], v[44:45] offset0:28 offset1:29
+; SI-NEXT: ds_write2_b64 v0, v[10:11], v[42:43] offset0:26 offset1:27
+; SI-NEXT: ds_write2_b64 v0, v[8:9], v[6:7] offset0:24 offset1:25
; SI-NEXT: ds_write2_b64 v0, v[4:5], v[2:3] offset0:22 offset1:23
-; SI-NEXT: ds_write2_b64 v0, v[24:25], v[32:33] offset0:20 offset1:21
-; SI-NEXT: ds_write2_b64 v0, v[30:31], v[28:29] offset0:18 offset1:19
+; SI-NEXT: ds_write2_b64 v0, v[20:21], v[28:29] offset0:20 offset1:21
+; SI-NEXT: ds_write2_b64 v0, v[26:27], v[24:25] offset0:18 offset1:19
; SI-NEXT: buffer_load_dword v1, off, s[12:15], 0 ; 4-byte Folded Reload
; SI-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; SI-NEXT: s_waitcnt vmcnt(0)
-; SI-NEXT: ds_write2_b64 v0, v[26:27], v[1:2] offset0:16 offset1:17
+; SI-NEXT: ds_write2_b64 v0, v[22:23], v[1:2] offset0:16 offset1:17
; SI-NEXT: s_endpgm
;
; VI-NO-DS128-LABEL: local_zextload_v64i16_to_v64i32:
@@ -3177,111 +3181,108 @@ define amdgpu_kernel void @local_zextload_v64i16_to_v64i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; VI-NO-DS128-NEXT: s_mov_b32 s90, -1
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v16, s1
-; VI-NO-DS128-NEXT: ds_read2_b64 v[10:13], v16 offset1:1
-; VI-NO-DS128-NEXT: ds_read2_b64 v[17:20], v16 offset0:2 offset1:3
-; VI-NO-DS128-NEXT: ds_read2_b64 v[21:24], v16 offset0:4 offset1:5
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v24, s1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[25:28], v24 offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_read2_b64 v[29:32], v24 offset0:8 offset1:9
+; VI-NO-DS128-NEXT: ds_read2_b64 v[10:13], v24 offset1:1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[14:17], v24 offset0:2 offset1:3
+; VI-NO-DS128-NEXT: ds_read2_b64 v[20:23], v24 offset0:4 offset1:5
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v34, 16, v26
+; VI-NO-DS128-NEXT: v_and_b32_e32 v33, 0xffff, v26
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v36, 16, v25
+; VI-NO-DS128-NEXT: v_and_b32_e32 v35, 0xffff, v25
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v38, 16, v28
+; VI-NO-DS128-NEXT: v_and_b32_e32 v37, 0xffff, v28
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v40, 16, v27
+; VI-NO-DS128-NEXT: v_and_b32_e32 v39, 0xffff, v27
+; VI-NO-DS128-NEXT: ds_read2_b64 v[25:28], v24 offset0:10 offset1:11
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v42, 16, v30
+; VI-NO-DS128-NEXT: v_and_b32_e32 v41, 0xffff, v30
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v44, 16, v29
+; VI-NO-DS128-NEXT: v_and_b32_e32 v43, 0xffff, v29
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v46, 16, v32
+; VI-NO-DS128-NEXT: v_and_b32_e32 v45, 0xffff, v32
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v48, 16, v31
+; VI-NO-DS128-NEXT: v_and_b32_e32 v47, 0xffff, v31
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v50, 16, v26
+; VI-NO-DS128-NEXT: v_and_b32_e32 v49, 0xffff, v26
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v52, 16, v25
+; VI-NO-DS128-NEXT: v_and_b32_e32 v51, 0xffff, v25
+; VI-NO-DS128-NEXT: ds_read2_b64 v[29:32], v24 offset0:12 offset1:13
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v56, 16, v27
+; VI-NO-DS128-NEXT: v_and_b32_e32 v55, 0xffff, v27
+; VI-NO-DS128-NEXT: ds_read2_b64 v[24:27], v24 offset0:14 offset1:15
; VI-NO-DS128-NEXT: s_mov_b32 s91, 0xe80000
; VI-NO-DS128-NEXT: s_add_u32 s88, s88, s11
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(2)
+; VI-NO-DS128-NEXT: s_addc_u32 s89, s89, 0
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v11
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v10
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v13
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v12
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v18
; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v11
-; VI-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v10
-; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v13
-; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v12
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v17
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v20
-; VI-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v18
-; VI-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v17
-; VI-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v20
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v19
-; VI-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v19
-; VI-NO-DS128-NEXT: ds_read2_b64 v[17:20], v16 offset0:6 offset1:7
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v26, 16, v22
-; VI-NO-DS128-NEXT: v_and_b32_e32 v25, 0xffff, v22
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v28, 16, v21
-; VI-NO-DS128-NEXT: v_and_b32_e32 v27, 0xffff, v21
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v30, 16, v24
-; VI-NO-DS128-NEXT: v_and_b32_e32 v29, 0xffff, v24
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v32, 16, v23
-; VI-NO-DS128-NEXT: v_and_b32_e32 v31, 0xffff, v23
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v34, 16, v18
-; VI-NO-DS128-NEXT: v_and_b32_e32 v33, 0xffff, v18
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v36, 16, v17
-; VI-NO-DS128-NEXT: v_and_b32_e32 v35, 0xffff, v17
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v38, 16, v20
-; VI-NO-DS128-NEXT: ds_read2_b64 v[21:24], v16 offset0:8 offset1:9
-; VI-NO-DS128-NEXT: v_and_b32_e32 v37, 0xffff, v20
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v40, 16, v19
-; VI-NO-DS128-NEXT: v_and_b32_e32 v39, 0xffff, v19
-; VI-NO-DS128-NEXT: ds_read2_b64 v[17:20], v16 offset0:10 offset1:11
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v42, 16, v22
-; VI-NO-DS128-NEXT: v_and_b32_e32 v41, 0xffff, v22
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v44, 16, v21
-; VI-NO-DS128-NEXT: v_and_b32_e32 v43, 0xffff, v21
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v46, 16, v24
-; VI-NO-DS128-NEXT: v_and_b32_e32 v45, 0xffff, v24
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v48, 16, v23
-; VI-NO-DS128-NEXT: v_and_b32_e32 v47, 0xffff, v23
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v50, 16, v18
-; VI-NO-DS128-NEXT: v_and_b32_e32 v49, 0xffff, v18
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v52, 16, v17
-; VI-NO-DS128-NEXT: v_and_b32_e32 v51, 0xffff, v17
-; VI-NO-DS128-NEXT: ds_read2_b64 v[21:24], v16 offset0:12 offset1:13
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v56, 16, v19
-; VI-NO-DS128-NEXT: v_and_b32_e32 v55, 0xffff, v19
-; VI-NO-DS128-NEXT: ds_read2_b64 v[16:19], v16 offset0:14 offset1:15
-; VI-NO-DS128-NEXT: s_addc_u32 s89, s89, 0
; VI-NO-DS128-NEXT: buffer_store_dword v0, off, s[88:91], 0 ; 4-byte Folded Spill
; VI-NO-DS128-NEXT: buffer_store_dword v1, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v54, 16, v20
-; VI-NO-DS128-NEXT: v_and_b32_e32 v53, 0xffff, v20
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v54, 16, v28
+; VI-NO-DS128-NEXT: v_and_b32_e32 v53, 0xffff, v28
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v19
-; VI-NO-DS128-NEXT: v_and_b32_e32 v19, 0xffff, v19
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v18
-; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v18
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v18, s0
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v58, 16, v22
-; VI-NO-DS128-NEXT: v_and_b32_e32 v57, 0xffff, v22
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v21
-; VI-NO-DS128-NEXT: v_and_b32_e32 v21, 0xffff, v21
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v60, 16, v24
-; VI-NO-DS128-NEXT: v_and_b32_e32 v59, 0xffff, v24
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v24, 16, v23
-; VI-NO-DS128-NEXT: v_and_b32_e32 v23, 0xffff, v23
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v62, 16, v17
-; VI-NO-DS128-NEXT: v_and_b32_e32 v61, 0xffff, v17
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v16
-; VI-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[0:1], v[19:20] offset0:30 offset1:31
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[16:17], v[61:62] offset0:28 offset1:29
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[23:24], v[59:60] offset0:26 offset1:27
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[21:22], v[57:58] offset0:24 offset1:25
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[55:56], v[53:54] offset0:22 offset1:23
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[51:52], v[49:50] offset0:20 offset1:21
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[47:48], v[45:46] offset0:18 offset1:19
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[43:44], v[41:42] offset0:16 offset1:17
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[39:40], v[37:38] offset0:14 offset1:15
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[35:36], v[33:34] offset0:12 offset1:13
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[31:32], v[29:30] offset0:10 offset1:11
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[27:28], v[25:26] offset0:8 offset1:9
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[14:15], v[12:13] offset0:6 offset1:7
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[10:11], v[8:9] offset0:4 offset1:5
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[6:7], v[4:5] offset0:2 offset1:3
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v28, 16, v27
+; VI-NO-DS128-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v26
+; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v26
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v26, s0
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[0:1], v[27:28] offset0:30 offset1:31
; VI-NO-DS128-NEXT: buffer_load_dword v0, off, s[88:91], 0 ; 4-byte Folded Reload
; VI-NO-DS128-NEXT: buffer_load_dword v1, off, s[88:91], 0 offset:4 ; 4-byte Folded Reload
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v10
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v13
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v12
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v15
+; VI-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v10
+; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v13
+; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v12
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v14
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v17
+; VI-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v15
+; VI-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v14
+; VI-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v17
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v16
+; VI-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v21
+; VI-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v21
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v20
+; VI-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v20
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v23
+; VI-NO-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v23
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v22
+; VI-NO-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v22
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v58, 16, v30
+; VI-NO-DS128-NEXT: v_and_b32_e32 v57, 0xffff, v30
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v30, 16, v29
+; VI-NO-DS128-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v60, 16, v32
+; VI-NO-DS128-NEXT: v_and_b32_e32 v59, 0xffff, v32
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v32, 16, v31
+; VI-NO-DS128-NEXT: v_and_b32_e32 v31, 0xffff, v31
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v62, 16, v25
+; VI-NO-DS128-NEXT: v_and_b32_e32 v61, 0xffff, v25
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v24
+; VI-NO-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[24:25], v[61:62] offset0:28 offset1:29
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[31:32], v[59:60] offset0:26 offset1:27
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[29:30], v[57:58] offset0:24 offset1:25
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[55:56], v[53:54] offset0:22 offset1:23
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[51:52], v[49:50] offset0:20 offset1:21
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[47:48], v[45:46] offset0:18 offset1:19
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[43:44], v[41:42] offset0:16 offset1:17
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[39:40], v[37:38] offset0:14 offset1:15
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[35:36], v[33:34] offset0:12 offset1:13
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[22:23], v[20:21] offset0:10 offset1:11
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[18:19], v[16:17] offset0:8 offset1:9
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[14:15], v[12:13] offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[10:11], v[8:9] offset0:4 offset1:5
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[6:7], v[4:5] offset0:2 offset1:3
; VI-NO-DS128-NEXT: s_waitcnt vmcnt(0)
-; VI-NO-DS128-NEXT: ds_write2_b64 v18, v[2:3], v[0:1] offset1:1
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[2:3], v[0:1] offset1:1
; VI-NO-DS128-NEXT: s_endpgm
;
; GFX9-NO-DS128-LABEL: local_zextload_v64i16_to_v64i32:
@@ -3292,20 +3293,62 @@ define amdgpu_kernel void @local_zextload_v64i16_to_v64i32(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: s_mov_b32 s14, -1
; GFX9-NO-DS128-NEXT: s_mov_b32 s15, 0xe00000
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v56, s1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[10:13], v56 offset1:1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[14:17], v56 offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[18:21], v56 offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[22:25], v56 offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v24, s1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[25:28], v24 offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[29:32], v24 offset0:8 offset1:9
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[10:13], v24 offset1:1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[14:17], v24 offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[20:23], v24 offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v34, 16, v26
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v33, 0xffff, v26
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v36, 16, v25
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v35, 0xffff, v25
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v38, 16, v28
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v37, 0xffff, v28
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v40, 16, v27
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v39, 0xffff, v27
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[25:28], v24 offset0:10 offset1:11
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v42, 16, v30
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v41, 0xffff, v30
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v44, 16, v29
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v43, 0xffff, v29
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v46, 16, v32
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v45, 0xffff, v32
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v48, 16, v31
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v47, 0xffff, v31
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v50, 16, v26
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v49, 0xffff, v26
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v52, 16, v25
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v51, 0xffff, v25
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[29:32], v24 offset0:12 offset1:13
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v56, 16, v27
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v55, 0xffff, v27
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[24:27], v24 offset0:14 offset1:15
; GFX9-NO-DS128-NEXT: s_add_u32 s12, s12, s11
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(3)
+; GFX9-NO-DS128-NEXT: s_addc_u32 s13, s13, 0
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v11
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v11
+; GFX9-NO-DS128-NEXT: buffer_store_dword v0, off, s[12:15], 0 ; 4-byte Folded Spill
+; GFX9-NO-DS128-NEXT: s_nop 0
+; GFX9-NO-DS128-NEXT: buffer_store_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v54, 16, v28
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v53, 0xffff, v28
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v28, 16, v27
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v27, 0xffff, v27
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v26
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v26
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v26, s0
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[0:1], v[27:28] offset0:30 offset1:31
+; GFX9-NO-DS128-NEXT: buffer_load_dword v0, off, s[12:15], 0 ; 4-byte Folded Reload
+; GFX9-NO-DS128-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v10
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v13
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v12
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(2)
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v15
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v11
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v10
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v13
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v12
@@ -3316,87 +3359,42 @@ define amdgpu_kernel void @local_zextload_v64i16_to_v64i32(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v17
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v16
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v16
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v27, 16, v19
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v19
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v18
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v18
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v21
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v21
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v33, 16, v20
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v32, 0xffff, v20
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v35, 16, v23
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v34, 0xffff, v23
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v37, 16, v22
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v36, 0xffff, v22
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[16:19], v56 offset0:8 offset1:9
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[20:23], v56 offset0:10 offset1:11
-; GFX9-NO-DS128-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NO-DS128-NEXT: buffer_store_dword v0, off, s[12:15], 0 ; 4-byte Folded Spill
-; GFX9-NO-DS128-NEXT: s_nop 0
-; GFX9-NO-DS128-NEXT: buffer_store_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v41, 16, v17
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v40, 0xffff, v17
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v43, 16, v16
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v42, 0xffff, v16
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v45, 16, v19
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v44, 0xffff, v19
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v47, 16, v18
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v46, 0xffff, v18
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v49, 16, v21
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v48, 0xffff, v21
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v51, 16, v20
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v50, 0xffff, v20
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v53, 16, v23
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[16:19], v56 offset0:12 offset1:13
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v52, 0xffff, v23
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v55, 16, v22
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v54, 0xffff, v22
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[20:23], v56 offset0:14 offset1:15
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v39, 16, v25
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v38, 0xffff, v25
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v24
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v24
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v63, 16, v23
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v62, 0xffff, v23
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v21
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v21
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v20
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v20
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v23
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v23
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v22
; GFX9-NO-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v22
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v57, 16, v17
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v56, 0xffff, v17
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v16
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v16
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v59, 16, v19
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v58, 0xffff, v19
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v18
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v18
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v61, 16, v21
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v60, 0xffff, v21
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v20
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v20
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[22:23], v[62:63] offset0:30 offset1:31
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[20:21], v[60:61] offset0:28 offset1:29
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[18:19], v[58:59] offset0:26 offset1:27
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[16:17], v[56:57] offset0:24 offset1:25
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[54:55], v[52:53] offset0:22 offset1:23
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[50:51], v[48:49] offset0:20 offset1:21
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[46:47], v[44:45] offset0:18 offset1:19
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[42:43], v[40:41] offset0:16 offset1:17
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[24:25], v[38:39] offset0:14 offset1:15
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[36:37], v[34:35] offset0:12 offset1:13
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[32:33], v[30:31] offset0:10 offset1:11
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[28:29], v[26:27] offset0:8 offset1:9
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[14:15], v[12:13] offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[10:11], v[8:9] offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[6:7], v[4:5] offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: buffer_load_dword v4, off, s[12:15], 0 ; 4-byte Folded Reload
-; GFX9-NO-DS128-NEXT: buffer_load_dword v5, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v58, 16, v30
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v57, 0xffff, v30
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v30, 16, v29
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v29, 0xffff, v29
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v60, 16, v32
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v59, 0xffff, v32
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v32, 16, v31
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v31, 0xffff, v31
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v62, 16, v25
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v61, 0xffff, v25
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v24
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v24
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[24:25], v[61:62] offset0:28 offset1:29
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[31:32], v[59:60] offset0:26 offset1:27
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[29:30], v[57:58] offset0:24 offset1:25
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[55:56], v[53:54] offset0:22 offset1:23
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[51:52], v[49:50] offset0:20 offset1:21
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[47:48], v[45:46] offset0:18 offset1:19
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[43:44], v[41:42] offset0:16 offset1:17
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[39:40], v[37:38] offset0:14 offset1:15
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[35:36], v[33:34] offset0:12 offset1:13
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[22:23], v[20:21] offset0:10 offset1:11
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[18:19], v[16:17] offset0:8 offset1:9
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[14:15], v[12:13] offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[10:11], v[8:9] offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[6:7], v[4:5] offset0:2 offset1:3
; GFX9-NO-DS128-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[2:3], v[0:1] offset1:1
; GFX9-NO-DS128-NEXT: s_endpgm
;
; EG-LABEL: local_zextload_v64i16_to_v64i32:
@@ -3799,116 +3797,125 @@ define amdgpu_kernel void @local_zextload_v64i16_to_v64i32(ptr addrspace(3) %out
; VI-DS128-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; VI-DS128-NEXT: s_mov_b32 s90, -1
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_mov_b32_e32 v0, s1
-; VI-DS128-NEXT: ds_read_b128 v[8:11], v0
-; VI-DS128-NEXT: ds_read_b128 v[16:19], v0 offset:16
+; VI-DS128-NEXT: v_mov_b32_e32 v24, s1
+; VI-DS128-NEXT: ds_read_b128 v[8:11], v24
; VI-DS128-NEXT: s_mov_b32 s91, 0xe80000
+; VI-DS128-NEXT: ds_read_b128 v[58:61], v24 offset:96
; VI-DS128-NEXT: s_add_u32 s88, s88, s11
-; VI-DS128-NEXT: ds_read_b128 v[20:23], v0 offset:32
-; VI-DS128-NEXT: ds_read_b128 v[24:27], v0 offset:48
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v11
+; VI-DS128-NEXT: ds_read_b128 v[16:19], v24 offset:16
; VI-DS128-NEXT: s_addc_u32 s89, s89, 0
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v10
-; VI-DS128-NEXT: v_mov_b32_e32 v4, v3
-; VI-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v11
-; VI-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v10
-; VI-DS128-NEXT: buffer_store_dword v1, off, s[88:91], 0 ; 4-byte Folded Spill
-; VI-DS128-NEXT: buffer_store_dword v2, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
-; VI-DS128-NEXT: buffer_store_dword v3, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
-; VI-DS128-NEXT: buffer_store_dword v4, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v9
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v8
-; VI-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v9
-; VI-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v8
-; VI-DS128-NEXT: buffer_store_dword v4, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill
-; VI-DS128-NEXT: buffer_store_dword v5, off, s[88:91], 0 offset:20 ; 4-byte Folded Spill
-; VI-DS128-NEXT: buffer_store_dword v6, off, s[88:91], 0 offset:24 ; 4-byte Folded Spill
-; VI-DS128-NEXT: buffer_store_dword v7, off, s[88:91], 0 offset:28 ; 4-byte Folded Spill
; VI-DS128-NEXT: s_waitcnt lgkmcnt(2)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v19
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v17
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v16
-; VI-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v19
-; VI-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v18
-; VI-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v17
-; VI-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v16
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v23
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v22
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v21
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v20
-; VI-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v23
-; VI-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v22
-; VI-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v21
-; VI-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v20
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v27
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v26
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v35, 16, v25
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v33, 16, v24
-; VI-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v27
-; VI-DS128-NEXT: ds_read_b128 v[36:39], v0 offset:64
-; VI-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v26
-; VI-DS128-NEXT: v_and_b32_e32 v34, 0xffff, v25
-; VI-DS128-NEXT: v_and_b32_e32 v32, 0xffff, v24
-; VI-DS128-NEXT: ds_read_b128 v[24:27], v0 offset:80
-; VI-DS128-NEXT: ds_read_b128 v[55:58], v0 offset:96
-; VI-DS128-NEXT: buffer_store_dword v1, off, s[88:91], 0 offset:32 ; 4-byte Folded Spill
-; VI-DS128-NEXT: buffer_store_dword v2, off, s[88:91], 0 offset:36 ; 4-byte Folded Spill
-; VI-DS128-NEXT: buffer_store_dword v3, off, s[88:91], 0 offset:40 ; 4-byte Folded Spill
-; VI-DS128-NEXT: buffer_store_dword v4, off, s[88:91], 0 offset:44 ; 4-byte Folded Spill
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v11
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v10
+; VI-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v11
+; VI-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v10
+; VI-DS128-NEXT: buffer_store_dword v0, off, s[88:91], 0 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v1, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v2, off, s[88:91], 0 offset:8 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v3, off, s[88:91], 0 offset:12 ; 4-byte Folded Spill
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v8
+; VI-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v9
+; VI-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v8
+; VI-DS128-NEXT: buffer_store_dword v3, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v4, off, s[88:91], 0 offset:20 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v5, off, s[88:91], 0 offset:24 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v6, off, s[88:91], 0 offset:28 ; 4-byte Folded Spill
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v61
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v60
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v59
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v58
+; VI-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v61
+; VI-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v60
+; VI-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v59
+; VI-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v58
+; VI-DS128-NEXT: ds_read_b128 v[58:61], v24 offset:112
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v19
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v18
+; VI-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v19
+; VI-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v18
+; VI-DS128-NEXT: buffer_store_dword v0, off, s[88:91], 0 offset:32 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v1, off, s[88:91], 0 offset:36 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v2, off, s[88:91], 0 offset:40 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v3, off, s[88:91], 0 offset:44 ; 4-byte Folded Spill
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v17
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v16
+; VI-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v17
+; VI-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v16
+; VI-DS128-NEXT: buffer_store_dword v3, off, s[88:91], 0 offset:48 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v4, off, s[88:91], 0 offset:52 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v5, off, s[88:91], 0 offset:56 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v6, off, s[88:91], 0 offset:60 ; 4-byte Folded Spill
+; VI-DS128-NEXT: ds_read_b128 v[25:28], v24 offset:32
+; VI-DS128-NEXT: ds_read_b128 v[29:32], v24 offset:48
+; VI-DS128-NEXT: ds_read_b128 v[36:39], v24 offset:64
+; VI-DS128-NEXT: ds_read_b128 v[47:50], v24 offset:80
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v59
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v58
+; VI-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v59
+; VI-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v58
+; VI-DS128-NEXT: v_mov_b32_e32 v24, s0
+; VI-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:224
+; VI-DS128-NEXT: buffer_load_dword v0, off, s[88:91], 0 offset:48 ; 4-byte Folded Reload
+; VI-DS128-NEXT: buffer_load_dword v1, off, s[88:91], 0 offset:52 ; 4-byte Folded Reload
+; VI-DS128-NEXT: buffer_load_dword v2, off, s[88:91], 0 offset:56 ; 4-byte Folded Reload
+; VI-DS128-NEXT: buffer_load_dword v3, off, s[88:91], 0 offset:60 ; 4-byte Folded Reload
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v28
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v27
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v26
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v25
+; VI-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v28
+; VI-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v27
+; VI-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v26
+; VI-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v25
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(3)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v28, 16, v32
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v26, 16, v31
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v35, 16, v30
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v33, 16, v29
+; VI-DS128-NEXT: v_and_b32_e32 v27, 0xffff, v32
+; VI-DS128-NEXT: v_and_b32_e32 v25, 0xffff, v31
+; VI-DS128-NEXT: v_and_b32_e32 v34, 0xffff, v30
+; VI-DS128-NEXT: v_and_b32_e32 v32, 0xffff, v29
; VI-DS128-NEXT: s_waitcnt lgkmcnt(2)
; VI-DS128-NEXT: v_lshrrev_b32_e32 v42, 16, v39
; VI-DS128-NEXT: v_lshrrev_b32_e32 v40, 16, v38
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v50, 16, v27
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v48, 16, v26
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v54, 16, v25
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v52, 16, v24
-; VI-DS128-NEXT: v_and_b32_e32 v49, 0xffff, v27
-; VI-DS128-NEXT: v_and_b32_e32 v47, 0xffff, v26
-; VI-DS128-NEXT: v_and_b32_e32 v53, 0xffff, v25
-; VI-DS128-NEXT: v_and_b32_e32 v51, 0xffff, v24
-; VI-DS128-NEXT: ds_read_b128 v[24:27], v0 offset:112
; VI-DS128-NEXT: v_lshrrev_b32_e32 v46, 16, v37
; VI-DS128-NEXT: v_lshrrev_b32_e32 v44, 16, v36
; VI-DS128-NEXT: v_and_b32_e32 v41, 0xffff, v39
; VI-DS128-NEXT: v_and_b32_e32 v39, 0xffff, v38
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v25
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v24
-; VI-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v25
-; VI-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v24
-; VI-DS128-NEXT: v_mov_b32_e32 v24, s0
; VI-DS128-NEXT: v_and_b32_e32 v45, 0xffff, v37
; VI-DS128-NEXT: v_and_b32_e32 v43, 0xffff, v36
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v61, 16, v58
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v59, 16, v57
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v56
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v55
-; VI-DS128-NEXT: v_and_b32_e32 v60, 0xffff, v58
-; VI-DS128-NEXT: v_and_b32_e32 v58, 0xffff, v57
-; VI-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v56
-; VI-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v55
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v27
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v26
-; VI-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v27
-; VI-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v26
-; VI-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:224
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v53, 16, v50
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v51, 16, v49
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v57, 16, v48
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v55, 16, v47
+; VI-DS128-NEXT: v_and_b32_e32 v52, 0xffff, v50
+; VI-DS128-NEXT: v_and_b32_e32 v50, 0xffff, v49
+; VI-DS128-NEXT: v_and_b32_e32 v56, 0xffff, v48
+; VI-DS128-NEXT: v_and_b32_e32 v54, 0xffff, v47
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v61
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v60
+; VI-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v61
+; VI-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v60
; VI-DS128-NEXT: ds_write_b128 v24, v[4:7] offset:240
; VI-DS128-NEXT: ds_write_b128 v24, v[8:11] offset:192
-; VI-DS128-NEXT: ds_write_b128 v24, v[58:61] offset:208
-; VI-DS128-NEXT: ds_write_b128 v24, v[51:54] offset:160
-; VI-DS128-NEXT: ds_write_b128 v24, v[47:50] offset:176
+; VI-DS128-NEXT: ds_write_b128 v24, v[12:15] offset:208
+; VI-DS128-NEXT: ds_write_b128 v24, v[54:57] offset:160
+; VI-DS128-NEXT: ds_write_b128 v24, v[50:53] offset:176
; VI-DS128-NEXT: ds_write_b128 v24, v[43:46] offset:128
; VI-DS128-NEXT: ds_write_b128 v24, v[39:42] offset:144
; VI-DS128-NEXT: ds_write_b128 v24, v[32:35] offset:96
-; VI-DS128-NEXT: ds_write_b128 v24, v[20:23] offset:112
-; VI-DS128-NEXT: ds_write_b128 v24, v[28:31] offset:64
+; VI-DS128-NEXT: ds_write_b128 v24, v[25:28] offset:112
+; VI-DS128-NEXT: ds_write_b128 v24, v[20:23] offset:64
; VI-DS128-NEXT: ds_write_b128 v24, v[16:19] offset:80
-; VI-DS128-NEXT: ds_write_b128 v24, v[12:15] offset:32
+; VI-DS128-NEXT: s_waitcnt vmcnt(0)
+; VI-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:32
; VI-DS128-NEXT: buffer_load_dword v0, off, s[88:91], 0 offset:32 ; 4-byte Folded Reload
; VI-DS128-NEXT: buffer_load_dword v1, off, s[88:91], 0 offset:36 ; 4-byte Folded Reload
; VI-DS128-NEXT: buffer_load_dword v2, off, s[88:91], 0 offset:40 ; 4-byte Folded Reload
@@ -3937,118 +3944,128 @@ define amdgpu_kernel void @local_zextload_v64i16_to_v64i32(ptr addrspace(3) %out
; GFX9-DS128-NEXT: s_mov_b32 s14, -1
; GFX9-DS128-NEXT: s_mov_b32 s15, 0xe00000
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_mov_b32_e32 v0, s1
-; GFX9-DS128-NEXT: ds_read_b128 v[8:11], v0
-; GFX9-DS128-NEXT: ds_read_b128 v[16:19], v0 offset:16
+; GFX9-DS128-NEXT: v_mov_b32_e32 v24, s1
+; GFX9-DS128-NEXT: ds_read_b128 v[8:11], v24
+; GFX9-DS128-NEXT: ds_read_b128 v[58:61], v24 offset:96
; GFX9-DS128-NEXT: s_add_u32 s12, s12, s11
-; GFX9-DS128-NEXT: ds_read_b128 v[20:23], v0 offset:32
-; GFX9-DS128-NEXT: ds_read_b128 v[24:27], v0 offset:48
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v11
+; GFX9-DS128-NEXT: ds_read_b128 v[16:19], v24 offset:16
; GFX9-DS128-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v10
-; GFX9-DS128-NEXT: v_mov_b32_e32 v4, v3
-; GFX9-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v11
-; GFX9-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v10
-; GFX9-DS128-NEXT: buffer_store_dword v1, off, s[12:15], 0 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(2)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v11
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v10
+; GFX9-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v11
+; GFX9-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v10
+; GFX9-DS128-NEXT: buffer_store_dword v0, off, s[12:15], 0 ; 4-byte Folded Spill
; GFX9-DS128-NEXT: s_nop 0
-; GFX9-DS128-NEXT: buffer_store_dword v2, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
-; GFX9-DS128-NEXT: buffer_store_dword v3, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
-; GFX9-DS128-NEXT: buffer_store_dword v4, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v9
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v8
-; GFX9-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v9
-; GFX9-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v8
-; GFX9-DS128-NEXT: buffer_store_dword v4, off, s[12:15], 0 offset:16 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v2, off, s[12:15], 0 offset:8 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v3, off, s[12:15], 0 offset:12 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v9
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v8
+; GFX9-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v9
+; GFX9-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v8
+; GFX9-DS128-NEXT: buffer_store_dword v3, off, s[12:15], 0 offset:16 ; 4-byte Folded Spill
; GFX9-DS128-NEXT: s_nop 0
-; GFX9-DS128-NEXT: buffer_store_dword v5, off, s[12:15], 0 offset:20 ; 4-byte Folded Spill
-; GFX9-DS128-NEXT: buffer_store_dword v6, off, s[12:15], 0 offset:24 ; 4-byte Folded Spill
-; GFX9-DS128-NEXT: buffer_store_dword v7, off, s[12:15], 0 offset:28 ; 4-byte Folded Spill
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(2)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v19
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v18
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v17
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v16
-; GFX9-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v19
-; GFX9-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v18
-; GFX9-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v17
-; GFX9-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v16
+; GFX9-DS128-NEXT: buffer_store_dword v4, off, s[12:15], 0 offset:20 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v5, off, s[12:15], 0 offset:24 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v6, off, s[12:15], 0 offset:28 ; 4-byte Folded Spill
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v23
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v22
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v21
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v29, 16, v20
-; GFX9-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v23
-; GFX9-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v22
-; GFX9-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v21
-; GFX9-DS128-NEXT: v_and_b32_e32 v28, 0xffff, v20
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v27
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v26
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v35, 16, v25
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v33, 16, v24
-; GFX9-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v27
-; GFX9-DS128-NEXT: ds_read_b128 v[36:39], v0 offset:64
-; GFX9-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v26
-; GFX9-DS128-NEXT: v_and_b32_e32 v34, 0xffff, v25
-; GFX9-DS128-NEXT: v_and_b32_e32 v32, 0xffff, v24
-; GFX9-DS128-NEXT: ds_read_b128 v[24:27], v0 offset:80
-; GFX9-DS128-NEXT: ds_read_b128 v[55:58], v0 offset:96
-; GFX9-DS128-NEXT: buffer_store_dword v1, off, s[12:15], 0 offset:32 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v61
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v60
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v59
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v58
+; GFX9-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v61
+; GFX9-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v60
+; GFX9-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v59
+; GFX9-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v58
+; GFX9-DS128-NEXT: ds_read_b128 v[58:61], v24 offset:112
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v19
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v18
+; GFX9-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v19
+; GFX9-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v18
+; GFX9-DS128-NEXT: buffer_store_dword v0, off, s[12:15], 0 offset:32 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: s_nop 0
+; GFX9-DS128-NEXT: buffer_store_dword v1, off, s[12:15], 0 offset:36 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v2, off, s[12:15], 0 offset:40 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v3, off, s[12:15], 0 offset:44 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v17
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v16
+; GFX9-DS128-NEXT: v_and_b32_e32 v5, 0xffff, v17
+; GFX9-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v16
+; GFX9-DS128-NEXT: buffer_store_dword v3, off, s[12:15], 0 offset:48 ; 4-byte Folded Spill
; GFX9-DS128-NEXT: s_nop 0
-; GFX9-DS128-NEXT: buffer_store_dword v2, off, s[12:15], 0 offset:36 ; 4-byte Folded Spill
-; GFX9-DS128-NEXT: buffer_store_dword v3, off, s[12:15], 0 offset:40 ; 4-byte Folded Spill
-; GFX9-DS128-NEXT: buffer_store_dword v4, off, s[12:15], 0 offset:44 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v4, off, s[12:15], 0 offset:52 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v5, off, s[12:15], 0 offset:56 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v6, off, s[12:15], 0 offset:60 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: ds_read_b128 v[25:28], v24 offset:32
+; GFX9-DS128-NEXT: ds_read_b128 v[29:32], v24 offset:48
+; GFX9-DS128-NEXT: ds_read_b128 v[36:39], v24 offset:64
+; GFX9-DS128-NEXT: ds_read_b128 v[47:50], v24 offset:80
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v59
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v58
+; GFX9-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v59
+; GFX9-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v58
+; GFX9-DS128-NEXT: v_mov_b32_e32 v24, s0
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:224
+; GFX9-DS128-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:48 ; 4-byte Folded Reload
+; GFX9-DS128-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:52 ; 4-byte Folded Reload
+; GFX9-DS128-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:56 ; 4-byte Folded Reload
+; GFX9-DS128-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:60 ; 4-byte Folded Reload
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v28
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v27
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v26
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v21, 16, v25
+; GFX9-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v28
+; GFX9-DS128-NEXT: v_and_b32_e32 v16, 0xffff, v27
+; GFX9-DS128-NEXT: v_and_b32_e32 v22, 0xffff, v26
+; GFX9-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v25
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(3)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v28, 16, v32
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v26, 16, v31
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v35, 16, v30
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v33, 16, v29
+; GFX9-DS128-NEXT: v_and_b32_e32 v27, 0xffff, v32
+; GFX9-DS128-NEXT: v_and_b32_e32 v25, 0xffff, v31
+; GFX9-DS128-NEXT: v_and_b32_e32 v34, 0xffff, v30
+; GFX9-DS128-NEXT: v_and_b32_e32 v32, 0xffff, v29
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(2)
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v42, 16, v39
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v40, 16, v38
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v50, 16, v27
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v48, 16, v26
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v54, 16, v25
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v52, 16, v24
-; GFX9-DS128-NEXT: v_and_b32_e32 v49, 0xffff, v27
-; GFX9-DS128-NEXT: v_and_b32_e32 v47, 0xffff, v26
-; GFX9-DS128-NEXT: v_and_b32_e32 v53, 0xffff, v25
-; GFX9-DS128-NEXT: v_and_b32_e32 v51, 0xffff, v24
-; GFX9-DS128-NEXT: ds_read_b128 v[24:27], v0 offset:112
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v46, 16, v37
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v44, 16, v36
; GFX9-DS128-NEXT: v_and_b32_e32 v41, 0xffff, v39
; GFX9-DS128-NEXT: v_and_b32_e32 v39, 0xffff, v38
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v25
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v24
-; GFX9-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v25
-; GFX9-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v24
-; GFX9-DS128-NEXT: v_mov_b32_e32 v24, s0
; GFX9-DS128-NEXT: v_and_b32_e32 v45, 0xffff, v37
; GFX9-DS128-NEXT: v_and_b32_e32 v43, 0xffff, v36
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v61, 16, v58
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v59, 16, v57
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v56
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v55
-; GFX9-DS128-NEXT: v_and_b32_e32 v60, 0xffff, v58
-; GFX9-DS128-NEXT: v_and_b32_e32 v58, 0xffff, v57
-; GFX9-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v56
-; GFX9-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v55
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v27
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v26
-; GFX9-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v27
-; GFX9-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v26
-; GFX9-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:224
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v53, 16, v50
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v51, 16, v49
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v57, 16, v48
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v55, 16, v47
+; GFX9-DS128-NEXT: v_and_b32_e32 v52, 0xffff, v50
+; GFX9-DS128-NEXT: v_and_b32_e32 v50, 0xffff, v49
+; GFX9-DS128-NEXT: v_and_b32_e32 v56, 0xffff, v48
+; GFX9-DS128-NEXT: v_and_b32_e32 v54, 0xffff, v47
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v61
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v60
+; GFX9-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v61
+; GFX9-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v60
; GFX9-DS128-NEXT: ds_write_b128 v24, v[4:7] offset:240
; GFX9-DS128-NEXT: ds_write_b128 v24, v[8:11] offset:192
-; GFX9-DS128-NEXT: ds_write_b128 v24, v[58:61] offset:208
-; GFX9-DS128-NEXT: ds_write_b128 v24, v[51:54] offset:160
-; GFX9-DS128-NEXT: ds_write_b128 v24, v[47:50] offset:176
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[12:15] offset:208
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[54:57] offset:160
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[50:53] offset:176
; GFX9-DS128-NEXT: ds_write_b128 v24, v[43:46] offset:128
; GFX9-DS128-NEXT: ds_write_b128 v24, v[39:42] offset:144
; GFX9-DS128-NEXT: ds_write_b128 v24, v[32:35] offset:96
-; GFX9-DS128-NEXT: ds_write_b128 v24, v[20:23] offset:112
-; GFX9-DS128-NEXT: ds_write_b128 v24, v[28:31] offset:64
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[25:28] offset:112
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[20:23] offset:64
; GFX9-DS128-NEXT: ds_write_b128 v24, v[16:19] offset:80
-; GFX9-DS128-NEXT: ds_write_b128 v24, v[12:15] offset:32
+; GFX9-DS128-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:32
; GFX9-DS128-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:32 ; 4-byte Folded Reload
; GFX9-DS128-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:36 ; 4-byte Folded Reload
; GFX9-DS128-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:40 ; 4-byte Folded Reload
@@ -4200,55 +4217,58 @@ define amdgpu_kernel void @local_sextload_v64i16_to_v64i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; VI-NO-DS128-NEXT: s_mov_b32 s90, -1
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v28, s1
-; VI-NO-DS128-NEXT: ds_read2_b64 v[20:23], v28 offset0:4 offset1:5
-; VI-NO-DS128-NEXT: ds_read2_b64 v[29:32], v28 offset0:6 offset1:7
-; VI-NO-DS128-NEXT: ds_read2_b64 v[33:36], v28 offset0:8 offset1:9
-; VI-NO-DS128-NEXT: ds_read2_b64 v[10:13], v28 offset1:1
-; VI-NO-DS128-NEXT: ds_read2_b64 v[14:17], v28 offset0:2 offset1:3
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v24, s1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[25:28], v24 offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_read2_b64 v[29:32], v24 offset0:8 offset1:9
+; VI-NO-DS128-NEXT: ds_read2_b64 v[10:13], v24 offset1:1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[14:17], v24 offset0:2 offset1:3
+; VI-NO-DS128-NEXT: ds_read2_b64 v[20:23], v24 offset0:4 offset1:5
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v34, 16, v26
+; VI-NO-DS128-NEXT: v_bfe_i32 v33, v26, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v36, 16, v25
+; VI-NO-DS128-NEXT: v_bfe_i32 v35, v25, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v38, 16, v28
+; VI-NO-DS128-NEXT: v_bfe_i32 v37, v28, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v40, 16, v27
+; VI-NO-DS128-NEXT: v_bfe_i32 v39, v27, 0, 16
+; VI-NO-DS128-NEXT: ds_read2_b64 v[25:28], v24 offset0:10 offset1:11
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v42, 16, v30
+; VI-NO-DS128-NEXT: v_bfe_i32 v41, v30, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v44, 16, v29
+; VI-NO-DS128-NEXT: v_bfe_i32 v43, v29, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v46, 16, v32
+; VI-NO-DS128-NEXT: v_bfe_i32 v45, v32, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v48, 16, v31
+; VI-NO-DS128-NEXT: v_bfe_i32 v47, v31, 0, 16
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v50, 16, v26
+; VI-NO-DS128-NEXT: v_bfe_i32 v49, v26, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v52, 16, v25
+; VI-NO-DS128-NEXT: v_bfe_i32 v51, v25, 0, 16
+; VI-NO-DS128-NEXT: ds_read2_b64 v[29:32], v24 offset0:12 offset1:13
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v56, 16, v27
+; VI-NO-DS128-NEXT: v_bfe_i32 v55, v27, 0, 16
+; VI-NO-DS128-NEXT: ds_read2_b64 v[24:27], v24 offset0:14 offset1:15
; VI-NO-DS128-NEXT: s_mov_b32 s91, 0xe80000
; VI-NO-DS128-NEXT: s_add_u32 s88, s88, s11
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v25, 16, v30
-; VI-NO-DS128-NEXT: v_bfe_i32 v24, v30, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v27, 16, v29
-; VI-NO-DS128-NEXT: v_bfe_i32 v26, v29, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v38, 16, v32
-; VI-NO-DS128-NEXT: v_bfe_i32 v37, v32, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v40, 16, v31
-; VI-NO-DS128-NEXT: v_bfe_i32 v39, v31, 0, 16
-; VI-NO-DS128-NEXT: ds_read2_b64 v[29:32], v28 offset0:10 offset1:11
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v42, 16, v34
-; VI-NO-DS128-NEXT: v_bfe_i32 v41, v34, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v44, 16, v33
-; VI-NO-DS128-NEXT: v_bfe_i32 v43, v33, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v46, 16, v36
-; VI-NO-DS128-NEXT: v_bfe_i32 v45, v36, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v48, 16, v35
-; VI-NO-DS128-NEXT: v_bfe_i32 v47, v35, 0, 16
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v50, 16, v30
-; VI-NO-DS128-NEXT: v_bfe_i32 v49, v30, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v52, 16, v29
-; VI-NO-DS128-NEXT: v_bfe_i32 v51, v29, 0, 16
-; VI-NO-DS128-NEXT: ds_read2_b64 v[33:36], v28 offset0:12 offset1:13
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v56, 16, v31
-; VI-NO-DS128-NEXT: v_bfe_i32 v55, v31, 0, 16
-; VI-NO-DS128-NEXT: ds_read2_b64 v[28:31], v28 offset0:14 offset1:15
; VI-NO-DS128-NEXT: s_addc_u32 s89, s89, 0
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v11
; VI-NO-DS128-NEXT: v_bfe_i32 v0, v11, 0, 16
; VI-NO-DS128-NEXT: buffer_store_dword v0, off, s[88:91], 0 ; 4-byte Folded Spill
; VI-NO-DS128-NEXT: buffer_store_dword v1, off, s[88:91], 0 offset:4 ; 4-byte Folded Spill
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v54, 16, v32
-; VI-NO-DS128-NEXT: v_bfe_i32 v53, v32, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v54, 16, v28
+; VI-NO-DS128-NEXT: v_bfe_i32 v53, v28, 0, 16
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v32, 16, v31
-; VI-NO-DS128-NEXT: v_bfe_i32 v31, v31, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v30
-; VI-NO-DS128-NEXT: v_bfe_i32 v0, v30, 0, 16
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v30, s0
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v28, 16, v27
+; VI-NO-DS128-NEXT: v_bfe_i32 v27, v27, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v26
+; VI-NO-DS128-NEXT: v_bfe_i32 v0, v26, 0, 16
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v26, s0
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[0:1], v[27:28] offset0:30 offset1:31
+; VI-NO-DS128-NEXT: buffer_load_dword v0, off, s[88:91], 0 ; 4-byte Folded Reload
+; VI-NO-DS128-NEXT: buffer_load_dword v1, off, s[88:91], 0 offset:4 ; 4-byte Folded Reload
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v10
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v13
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v12
@@ -4271,37 +4291,34 @@ define amdgpu_kernel void @local_sextload_v64i16_to_v64i32(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: v_bfe_i32 v20, v23, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v23, 16, v22
; VI-NO-DS128-NEXT: v_bfe_i32 v22, v22, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v58, 16, v34
-; VI-NO-DS128-NEXT: v_bfe_i32 v57, v34, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v34, 16, v33
-; VI-NO-DS128-NEXT: v_bfe_i32 v33, v33, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v60, 16, v36
-; VI-NO-DS128-NEXT: v_bfe_i32 v59, v36, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v36, 16, v35
-; VI-NO-DS128-NEXT: v_bfe_i32 v35, v35, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v62, 16, v29
-; VI-NO-DS128-NEXT: v_bfe_i32 v61, v29, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v29, 16, v28
-; VI-NO-DS128-NEXT: v_bfe_i32 v28, v28, 0, 16
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[0:1], v[31:32] offset0:30 offset1:31
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[28:29], v[61:62] offset0:28 offset1:29
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[35:36], v[59:60] offset0:26 offset1:27
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[33:34], v[57:58] offset0:24 offset1:25
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[55:56], v[53:54] offset0:22 offset1:23
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[51:52], v[49:50] offset0:20 offset1:21
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[47:48], v[45:46] offset0:18 offset1:19
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[43:44], v[41:42] offset0:16 offset1:17
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[39:40], v[37:38] offset0:14 offset1:15
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[26:27], v[24:25] offset0:12 offset1:13
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[22:23], v[20:21] offset0:10 offset1:11
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[18:19], v[16:17] offset0:8 offset1:9
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[14:15], v[12:13] offset0:6 offset1:7
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[10:11], v[8:9] offset0:4 offset1:5
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[6:7], v[4:5] offset0:2 offset1:3
-; VI-NO-DS128-NEXT: buffer_load_dword v0, off, s[88:91], 0 ; 4-byte Folded Reload
-; VI-NO-DS128-NEXT: buffer_load_dword v1, off, s[88:91], 0 offset:4 ; 4-byte Folded Reload
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v58, 16, v30
+; VI-NO-DS128-NEXT: v_bfe_i32 v57, v30, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v30, 16, v29
+; VI-NO-DS128-NEXT: v_bfe_i32 v29, v29, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v60, 16, v32
+; VI-NO-DS128-NEXT: v_bfe_i32 v59, v32, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v32, 16, v31
+; VI-NO-DS128-NEXT: v_bfe_i32 v31, v31, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v62, 16, v25
+; VI-NO-DS128-NEXT: v_bfe_i32 v61, v25, 0, 16
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v25, 16, v24
+; VI-NO-DS128-NEXT: v_bfe_i32 v24, v24, 0, 16
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[24:25], v[61:62] offset0:28 offset1:29
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[31:32], v[59:60] offset0:26 offset1:27
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[29:30], v[57:58] offset0:24 offset1:25
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[55:56], v[53:54] offset0:22 offset1:23
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[51:52], v[49:50] offset0:20 offset1:21
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[47:48], v[45:46] offset0:18 offset1:19
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[43:44], v[41:42] offset0:16 offset1:17
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[39:40], v[37:38] offset0:14 offset1:15
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[35:36], v[33:34] offset0:12 offset1:13
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[22:23], v[20:21] offset0:10 offset1:11
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[18:19], v[16:17] offset0:8 offset1:9
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[14:15], v[12:13] offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[10:11], v[8:9] offset0:4 offset1:5
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[6:7], v[4:5] offset0:2 offset1:3
; VI-NO-DS128-NEXT: s_waitcnt vmcnt(0)
-; VI-NO-DS128-NEXT: ds_write2_b64 v30, v[2:3], v[0:1] offset1:1
+; VI-NO-DS128-NEXT: ds_write2_b64 v26, v[2:3], v[0:1] offset1:1
; VI-NO-DS128-NEXT: s_endpgm
;
; GFX9-NO-DS128-LABEL: local_sextload_v64i16_to_v64i32:
@@ -4312,55 +4329,58 @@ define amdgpu_kernel void @local_sextload_v64i16_to_v64i32(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: s_mov_b32 s14, -1
; GFX9-NO-DS128-NEXT: s_mov_b32 s15, 0xe00000
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v28, s1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[20:23], v28 offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[29:32], v28 offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[33:36], v28 offset0:8 offset1:9
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[10:13], v28 offset1:1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[14:17], v28 offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v24, s1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[25:28], v24 offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[29:32], v24 offset0:8 offset1:9
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[10:13], v24 offset1:1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[14:17], v24 offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[20:23], v24 offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v34, 16, v26
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v33, v26, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v36, 16, v25
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v35, v25, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v38, 16, v28
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v37, v28, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v40, 16, v27
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v39, v27, 0, 16
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[25:28], v24 offset0:10 offset1:11
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v42, 16, v30
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v41, v30, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v44, 16, v29
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v43, v29, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v46, 16, v32
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v45, v32, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v48, 16, v31
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v47, v31, 0, 16
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v50, 16, v26
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v49, v26, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v52, 16, v25
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v51, v25, 0, 16
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[29:32], v24 offset0:12 offset1:13
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v56, 16, v27
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v55, v27, 0, 16
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[24:27], v24 offset0:14 offset1:15
; GFX9-NO-DS128-NEXT: s_add_u32 s12, s12, s11
; GFX9-NO-DS128-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v25, 16, v30
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v24, v30, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v27, 16, v29
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v26, v29, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v38, 16, v32
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v37, v32, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v40, 16, v31
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v39, v31, 0, 16
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[29:32], v28 offset0:10 offset1:11
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v42, 16, v34
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v41, v34, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v44, 16, v33
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v43, v33, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v46, 16, v36
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v45, v36, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v48, 16, v35
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v47, v35, 0, 16
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v50, 16, v30
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v49, v30, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v52, 16, v29
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v51, v29, 0, 16
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[33:36], v28 offset0:12 offset1:13
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v56, 16, v31
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v55, v31, 0, 16
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[28:31], v28 offset0:14 offset1:15
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v11
; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v11, 0, 16
; GFX9-NO-DS128-NEXT: buffer_store_dword v0, off, s[12:15], 0 ; 4-byte Folded Spill
; GFX9-NO-DS128-NEXT: s_nop 0
; GFX9-NO-DS128-NEXT: buffer_store_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Spill
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v54, 16, v32
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v53, v32, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v54, 16, v28
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v53, v28, 0, 16
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v32, 16, v31
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v31, v31, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v30
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v30, 0, 16
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v30, s0
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v28, 16, v27
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v27, v27, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v26
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v26, 0, 16
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v26, s0
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[0:1], v[27:28] offset0:30 offset1:31
+; GFX9-NO-DS128-NEXT: buffer_load_dword v0, off, s[12:15], 0 ; 4-byte Folded Reload
+; GFX9-NO-DS128-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v10
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v13
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v12
@@ -4383,37 +4403,34 @@ define amdgpu_kernel void @local_sextload_v64i16_to_v64i32(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: v_bfe_i32 v20, v23, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v23, 16, v22
; GFX9-NO-DS128-NEXT: v_bfe_i32 v22, v22, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v58, 16, v34
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v57, v34, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v34, 16, v33
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v33, v33, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v60, 16, v36
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v59, v36, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v36, 16, v35
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v35, v35, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v62, 16, v29
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v61, v29, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v29, 16, v28
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v28, v28, 0, 16
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[0:1], v[31:32] offset0:30 offset1:31
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[28:29], v[61:62] offset0:28 offset1:29
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[35:36], v[59:60] offset0:26 offset1:27
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[33:34], v[57:58] offset0:24 offset1:25
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[55:56], v[53:54] offset0:22 offset1:23
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[51:52], v[49:50] offset0:20 offset1:21
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[47:48], v[45:46] offset0:18 offset1:19
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[43:44], v[41:42] offset0:16 offset1:17
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[39:40], v[37:38] offset0:14 offset1:15
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[26:27], v[24:25] offset0:12 offset1:13
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[22:23], v[20:21] offset0:10 offset1:11
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[18:19], v[16:17] offset0:8 offset1:9
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[14:15], v[12:13] offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[10:11], v[8:9] offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[6:7], v[4:5] offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: buffer_load_dword v0, off, s[12:15], 0 ; 4-byte Folded Reload
-; GFX9-NO-DS128-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v58, 16, v30
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v57, v30, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v30, 16, v29
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v29, v29, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v60, 16, v32
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v59, v32, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v32, 16, v31
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v31, v31, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v62, 16, v25
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v61, v25, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v25, 16, v24
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v24, v24, 0, 16
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[24:25], v[61:62] offset0:28 offset1:29
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[31:32], v[59:60] offset0:26 offset1:27
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[29:30], v[57:58] offset0:24 offset1:25
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[55:56], v[53:54] offset0:22 offset1:23
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[51:52], v[49:50] offset0:20 offset1:21
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[47:48], v[45:46] offset0:18 offset1:19
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[43:44], v[41:42] offset0:16 offset1:17
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[39:40], v[37:38] offset0:14 offset1:15
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[35:36], v[33:34] offset0:12 offset1:13
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[22:23], v[20:21] offset0:10 offset1:11
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[18:19], v[16:17] offset0:8 offset1:9
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[14:15], v[12:13] offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[10:11], v[8:9] offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[6:7], v[4:5] offset0:2 offset1:3
; GFX9-NO-DS128-NEXT: s_waitcnt vmcnt(0)
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v30, v[2:3], v[0:1] offset1:1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v26, v[2:3], v[0:1] offset1:1
; GFX9-NO-DS128-NEXT: s_endpgm
;
; EG-LABEL: local_sextload_v64i16_to_v64i32:
@@ -4845,20 +4862,19 @@ define amdgpu_kernel void @local_sextload_v64i16_to_v64i32(ptr addrspace(3) %out
; VI-DS128-LABEL: local_sextload_v64i16_to_v64i32:
; VI-DS128: ; %bb.0:
; VI-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; VI-DS128-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; VI-DS128-NEXT: s_mov_b32 m0, -1
+; VI-DS128-NEXT: s_mov_b32 s88, SCRATCH_RSRC_DWORD0
; VI-DS128-NEXT: s_mov_b32 s89, SCRATCH_RSRC_DWORD1
; VI-DS128-NEXT: s_mov_b32 s90, -1
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_mov_b32_e32 v32, s1
-; VI-DS128-NEXT: ds_read_b128 v[8:11], v32
-; VI-DS128-NEXT: ds_read_b128 v[16:19], v32 offset:16
-; VI-DS128-NEXT: ds_read_b128 v[24:27], v32 offset:32
-; VI-DS128-NEXT: ds_read_b128 v[33:36], v32 offset:48
+; VI-DS128-NEXT: v_mov_b32_e32 v24, s1
+; VI-DS128-NEXT: ds_read_b128 v[8:11], v24
; VI-DS128-NEXT: s_mov_b32 s91, 0xe80000
+; VI-DS128-NEXT: ds_read_b128 v[58:61], v24 offset:96
; VI-DS128-NEXT: s_add_u32 s88, s88, s11
+; VI-DS128-NEXT: ds_read_b128 v[16:19], v24 offset:16
; VI-DS128-NEXT: s_addc_u32 s89, s89, 0
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(3)
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(2)
; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v11
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v10
; VI-DS128-NEXT: v_bfe_i32 v2, v11, 0, 16
@@ -4871,103 +4887,123 @@ define amdgpu_kernel void @local_sextload_v64i16_to_v64i32(ptr addrspace(3) %out
; VI-DS128-NEXT: v_ashrrev_i32_e32 v4, 16, v8
; VI-DS128-NEXT: v_bfe_i32 v5, v9, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v3, v8, 0, 16
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(2)
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v19
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v18
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v17
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v13, 16, v16
-; VI-DS128-NEXT: v_bfe_i32 v10, v19, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v8, v18, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v14, v17, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v12, v16, 0, 16
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v27
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v26
-; VI-DS128-NEXT: v_bfe_i32 v18, v27, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v16, v26, 0, 16
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v27, 16, v36
-; VI-DS128-NEXT: v_bfe_i32 v26, v36, 0, 16
-; VI-DS128-NEXT: ds_read_b128 v[36:39], v32 offset:64
-; VI-DS128-NEXT: ds_read_b128 v[40:43], v32 offset:80
-; VI-DS128-NEXT: ds_read_b128 v[56:59], v32 offset:96
; VI-DS128-NEXT: buffer_store_dword v3, off, s[88:91], 0 offset:16 ; 4-byte Folded Spill
; VI-DS128-NEXT: buffer_store_dword v4, off, s[88:91], 0 offset:20 ; 4-byte Folded Spill
; VI-DS128-NEXT: buffer_store_dword v5, off, s[88:91], 0 offset:24 ; 4-byte Folded Spill
; VI-DS128-NEXT: buffer_store_dword v6, off, s[88:91], 0 offset:28 ; 4-byte Folded Spill
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v23, 16, v25
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v61
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v13, 16, v60
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v59
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v58
+; VI-DS128-NEXT: v_bfe_i32 v14, v61, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v12, v60, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v10, v59, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v8, v58, 0, 16
+; VI-DS128-NEXT: ds_read_b128 v[58:61], v24 offset:112
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v19
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v18
+; VI-DS128-NEXT: v_bfe_i32 v2, v19, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v0, v18, 0, 16
+; VI-DS128-NEXT: buffer_store_dword v0, off, s[88:91], 0 offset:32 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v1, off, s[88:91], 0 offset:36 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v2, off, s[88:91], 0 offset:40 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v3, off, s[88:91], 0 offset:44 ; 4-byte Folded Spill
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v6, 16, v17
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v4, 16, v16
+; VI-DS128-NEXT: v_bfe_i32 v5, v17, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v3, v16, 0, 16
+; VI-DS128-NEXT: buffer_store_dword v3, off, s[88:91], 0 offset:48 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v4, off, s[88:91], 0 offset:52 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v5, off, s[88:91], 0 offset:56 ; 4-byte Folded Spill
+; VI-DS128-NEXT: buffer_store_dword v6, off, s[88:91], 0 offset:60 ; 4-byte Folded Spill
+; VI-DS128-NEXT: ds_read_b128 v[25:28], v24 offset:32
+; VI-DS128-NEXT: ds_read_b128 v[29:32], v24 offset:48
+; VI-DS128-NEXT: ds_read_b128 v[36:39], v24 offset:64
+; VI-DS128-NEXT: ds_read_b128 v[47:50], v24 offset:80
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v59
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v58
+; VI-DS128-NEXT: v_bfe_i32 v2, v59, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v0, v58, 0, 16
+; VI-DS128-NEXT: v_mov_b32_e32 v24, s0
+; VI-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:224
+; VI-DS128-NEXT: buffer_load_dword v0, off, s[88:91], 0 offset:48 ; 4-byte Folded Reload
+; VI-DS128-NEXT: buffer_load_dword v1, off, s[88:91], 0 offset:52 ; 4-byte Folded Reload
+; VI-DS128-NEXT: buffer_load_dword v2, off, s[88:91], 0 offset:56 ; 4-byte Folded Reload
+; VI-DS128-NEXT: buffer_load_dword v3, off, s[88:91], 0 offset:60 ; 4-byte Folded Reload
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v28
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v27
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v23, 16, v26
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v21, 16, v25
+; VI-DS128-NEXT: v_bfe_i32 v18, v28, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v16, v27, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v22, v26, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v20, v25, 0, 16
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(3)
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v28, 16, v32
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v26, 16, v31
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v35, 16, v30
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v33, 16, v29
+; VI-DS128-NEXT: v_bfe_i32 v27, v32, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v25, v31, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v34, v30, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v32, v29, 0, 16
; VI-DS128-NEXT: s_waitcnt lgkmcnt(2)
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v47, 16, v39
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v45, 16, v38
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v51, 16, v37
-; VI-DS128-NEXT: v_bfe_i32 v46, v39, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v44, v38, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v50, v37, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v42, 16, v39
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v40, 16, v38
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v46, 16, v37
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v44, 16, v36
+; VI-DS128-NEXT: v_bfe_i32 v41, v39, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v39, v38, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v45, v37, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v43, v36, 0, 16
; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v53, 16, v40
-; VI-DS128-NEXT: v_bfe_i32 v52, v40, 0, 16
-; VI-DS128-NEXT: ds_read_b128 v[37:40], v32 offset:112
-; VI-DS128-NEXT: v_mov_b32_e32 v32, s0
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v21, 16, v24
-; VI-DS128-NEXT: v_bfe_i32 v22, v25, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v20, v24, 0, 16
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v38
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v37
-; VI-DS128-NEXT: v_bfe_i32 v2, v38, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v0, v37, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v25, 16, v35
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v31, 16, v34
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v29, 16, v33
-; VI-DS128-NEXT: v_bfe_i32 v24, v35, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v30, v34, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v28, v33, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v49, 16, v36
-; VI-DS128-NEXT: v_bfe_i32 v48, v36, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v36, 16, v43
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v34, 16, v42
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v55, 16, v41
-; VI-DS128-NEXT: v_bfe_i32 v35, v43, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v33, v42, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v54, v41, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v62, 16, v59
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v60, 16, v58
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v57
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v56
-; VI-DS128-NEXT: v_bfe_i32 v61, v59, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v59, v58, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v6, v57, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v4, v56, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v43, 16, v40
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v41, 16, v39
-; VI-DS128-NEXT: v_bfe_i32 v42, v40, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v40, v39, 0, 16
-; VI-DS128-NEXT: ds_write_b128 v32, v[0:3] offset:224
-; VI-DS128-NEXT: ds_write_b128 v32, v[40:43] offset:240
-; VI-DS128-NEXT: ds_write_b128 v32, v[4:7] offset:192
-; VI-DS128-NEXT: ds_write_b128 v32, v[59:62] offset:208
-; VI-DS128-NEXT: ds_write_b128 v32, v[52:55] offset:160
-; VI-DS128-NEXT: ds_write_b128 v32, v[33:36] offset:176
-; VI-DS128-NEXT: ds_write_b128 v32, v[48:51] offset:128
-; VI-DS128-NEXT: ds_write_b128 v32, v[44:47] offset:144
-; VI-DS128-NEXT: ds_write_b128 v32, v[28:31] offset:96
-; VI-DS128-NEXT: ds_write_b128 v32, v[24:27] offset:112
-; VI-DS128-NEXT: ds_write_b128 v32, v[20:23] offset:64
-; VI-DS128-NEXT: ds_write_b128 v32, v[16:19] offset:80
-; VI-DS128-NEXT: ds_write_b128 v32, v[12:15] offset:32
-; VI-DS128-NEXT: ds_write_b128 v32, v[8:11] offset:48
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v53, 16, v50
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v51, 16, v49
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v57, 16, v48
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v55, 16, v47
+; VI-DS128-NEXT: v_bfe_i32 v52, v50, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v50, v49, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v56, v48, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v54, v47, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v61
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v60
+; VI-DS128-NEXT: v_bfe_i32 v6, v61, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v4, v60, 0, 16
+; VI-DS128-NEXT: ds_write_b128 v24, v[4:7] offset:240
+; VI-DS128-NEXT: ds_write_b128 v24, v[8:11] offset:192
+; VI-DS128-NEXT: ds_write_b128 v24, v[12:15] offset:208
+; VI-DS128-NEXT: ds_write_b128 v24, v[54:57] offset:160
+; VI-DS128-NEXT: ds_write_b128 v24, v[50:53] offset:176
+; VI-DS128-NEXT: ds_write_b128 v24, v[43:46] offset:128
+; VI-DS128-NEXT: ds_write_b128 v24, v[39:42] offset:144
+; VI-DS128-NEXT: ds_write_b128 v24, v[32:35] offset:96
+; VI-DS128-NEXT: ds_write_b128 v24, v[25:28] offset:112
+; VI-DS128-NEXT: ds_write_b128 v24, v[20:23] offset:64
+; VI-DS128-NEXT: ds_write_b128 v24, v[16:19] offset:80
+; VI-DS128-NEXT: s_waitcnt vmcnt(0)
+; VI-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:32
+; VI-DS128-NEXT: buffer_load_dword v0, off, s[88:91], 0 offset:32 ; 4-byte Folded Reload
+; VI-DS128-NEXT: buffer_load_dword v1, off, s[88:91], 0 offset:36 ; 4-byte Folded Reload
+; VI-DS128-NEXT: buffer_load_dword v2, off, s[88:91], 0 offset:40 ; 4-byte Folded Reload
+; VI-DS128-NEXT: buffer_load_dword v3, off, s[88:91], 0 offset:44 ; 4-byte Folded Reload
+; VI-DS128-NEXT: s_waitcnt vmcnt(0)
+; VI-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:48
; VI-DS128-NEXT: buffer_load_dword v0, off, s[88:91], 0 offset:16 ; 4-byte Folded Reload
; VI-DS128-NEXT: buffer_load_dword v1, off, s[88:91], 0 offset:20 ; 4-byte Folded Reload
; VI-DS128-NEXT: buffer_load_dword v2, off, s[88:91], 0 offset:24 ; 4-byte Folded Reload
; VI-DS128-NEXT: buffer_load_dword v3, off, s[88:91], 0 offset:28 ; 4-byte Folded Reload
; VI-DS128-NEXT: s_waitcnt vmcnt(0)
-; VI-DS128-NEXT: ds_write_b128 v32, v[0:3]
+; VI-DS128-NEXT: ds_write_b128 v24, v[0:3]
; VI-DS128-NEXT: buffer_load_dword v0, off, s[88:91], 0 ; 4-byte Folded Reload
; VI-DS128-NEXT: buffer_load_dword v1, off, s[88:91], 0 offset:4 ; 4-byte Folded Reload
; VI-DS128-NEXT: buffer_load_dword v2, off, s[88:91], 0 offset:8 ; 4-byte Folded Reload
; VI-DS128-NEXT: buffer_load_dword v3, off, s[88:91], 0 offset:12 ; 4-byte Folded Reload
; VI-DS128-NEXT: s_waitcnt vmcnt(0)
-; VI-DS128-NEXT: ds_write_b128 v32, v[0:3] offset:16
+; VI-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:16
; VI-DS128-NEXT: s_endpgm
;
; GFX9-DS128-LABEL: local_sextload_v64i16_to_v64i32:
@@ -4978,14 +5014,13 @@ define amdgpu_kernel void @local_sextload_v64i16_to_v64i32(ptr addrspace(3) %out
; GFX9-DS128-NEXT: s_mov_b32 s14, -1
; GFX9-DS128-NEXT: s_mov_b32 s15, 0xe00000
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_mov_b32_e32 v32, s1
-; GFX9-DS128-NEXT: ds_read_b128 v[8:11], v32
-; GFX9-DS128-NEXT: ds_read_b128 v[16:19], v32 offset:16
-; GFX9-DS128-NEXT: ds_read_b128 v[24:27], v32 offset:32
-; GFX9-DS128-NEXT: ds_read_b128 v[33:36], v32 offset:48
+; GFX9-DS128-NEXT: v_mov_b32_e32 v24, s1
+; GFX9-DS128-NEXT: ds_read_b128 v[8:11], v24
+; GFX9-DS128-NEXT: ds_read_b128 v[58:61], v24 offset:96
; GFX9-DS128-NEXT: s_add_u32 s12, s12, s11
+; GFX9-DS128-NEXT: ds_read_b128 v[16:19], v24 offset:16
; GFX9-DS128-NEXT: s_addc_u32 s13, s13, 0
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(3)
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(2)
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v11
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v10
; GFX9-DS128-NEXT: v_bfe_i32 v2, v11, 0, 16
@@ -4999,104 +5034,126 @@ define amdgpu_kernel void @local_sextload_v64i16_to_v64i32(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v4, 16, v8
; GFX9-DS128-NEXT: v_bfe_i32 v5, v9, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v3, v8, 0, 16
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(2)
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v19
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v18
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v17
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 16, v16
-; GFX9-DS128-NEXT: v_bfe_i32 v10, v19, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v8, v18, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v14, v17, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v12, v16, 0, 16
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v27
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v26
-; GFX9-DS128-NEXT: v_bfe_i32 v18, v27, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v16, v26, 0, 16
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v27, 16, v36
-; GFX9-DS128-NEXT: v_bfe_i32 v26, v36, 0, 16
-; GFX9-DS128-NEXT: ds_read_b128 v[36:39], v32 offset:64
-; GFX9-DS128-NEXT: ds_read_b128 v[40:43], v32 offset:80
-; GFX9-DS128-NEXT: ds_read_b128 v[56:59], v32 offset:96
; GFX9-DS128-NEXT: buffer_store_dword v3, off, s[12:15], 0 offset:16 ; 4-byte Folded Spill
; GFX9-DS128-NEXT: s_nop 0
; GFX9-DS128-NEXT: buffer_store_dword v4, off, s[12:15], 0 offset:20 ; 4-byte Folded Spill
; GFX9-DS128-NEXT: buffer_store_dword v5, off, s[12:15], 0 offset:24 ; 4-byte Folded Spill
; GFX9-DS128-NEXT: buffer_store_dword v6, off, s[12:15], 0 offset:28 ; 4-byte Folded Spill
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v23, 16, v25
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 16, v61
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 16, v60
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v11, 16, v59
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v9, 16, v58
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v61, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v12, v60, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v10, v59, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v8, v58, 0, 16
+; GFX9-DS128-NEXT: ds_read_b128 v[58:61], v24 offset:112
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v19
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v18
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v19, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v18, 0, 16
+; GFX9-DS128-NEXT: buffer_store_dword v0, off, s[12:15], 0 offset:32 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: s_nop 0
+; GFX9-DS128-NEXT: buffer_store_dword v1, off, s[12:15], 0 offset:36 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v2, off, s[12:15], 0 offset:40 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v3, off, s[12:15], 0 offset:44 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v6, 16, v17
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v4, 16, v16
+; GFX9-DS128-NEXT: v_bfe_i32 v5, v17, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v3, v16, 0, 16
+; GFX9-DS128-NEXT: buffer_store_dword v3, off, s[12:15], 0 offset:48 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: s_nop 0
+; GFX9-DS128-NEXT: buffer_store_dword v4, off, s[12:15], 0 offset:52 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v5, off, s[12:15], 0 offset:56 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: buffer_store_dword v6, off, s[12:15], 0 offset:60 ; 4-byte Folded Spill
+; GFX9-DS128-NEXT: ds_read_b128 v[25:28], v24 offset:32
+; GFX9-DS128-NEXT: ds_read_b128 v[29:32], v24 offset:48
+; GFX9-DS128-NEXT: ds_read_b128 v[36:39], v24 offset:64
+; GFX9-DS128-NEXT: ds_read_b128 v[47:50], v24 offset:80
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v59
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v58
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v59, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v58, 0, 16
+; GFX9-DS128-NEXT: v_mov_b32_e32 v24, s0
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:224
+; GFX9-DS128-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:48 ; 4-byte Folded Reload
+; GFX9-DS128-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:52 ; 4-byte Folded Reload
+; GFX9-DS128-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:56 ; 4-byte Folded Reload
+; GFX9-DS128-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:60 ; 4-byte Folded Reload
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(4)
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v19, 16, v28
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 16, v27
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v23, 16, v26
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v21, 16, v25
+; GFX9-DS128-NEXT: v_bfe_i32 v18, v28, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v16, v27, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v22, v26, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v20, v25, 0, 16
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(3)
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v28, 16, v32
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v26, 16, v31
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v35, 16, v30
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v33, 16, v29
+; GFX9-DS128-NEXT: v_bfe_i32 v27, v32, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v25, v31, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v34, v30, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v32, v29, 0, 16
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(2)
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v47, 16, v39
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v45, 16, v38
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v51, 16, v37
-; GFX9-DS128-NEXT: v_bfe_i32 v46, v39, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v44, v38, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v50, v37, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v42, 16, v39
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v40, 16, v38
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v46, 16, v37
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v44, 16, v36
+; GFX9-DS128-NEXT: v_bfe_i32 v41, v39, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v39, v38, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v45, v37, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v43, v36, 0, 16
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v53, 16, v40
-; GFX9-DS128-NEXT: v_bfe_i32 v52, v40, 0, 16
-; GFX9-DS128-NEXT: ds_read_b128 v[37:40], v32 offset:112
-; GFX9-DS128-NEXT: v_mov_b32_e32 v32, s0
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v21, 16, v24
-; GFX9-DS128-NEXT: v_bfe_i32 v22, v25, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v20, v24, 0, 16
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 16, v38
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 16, v37
-; GFX9-DS128-NEXT: v_bfe_i32 v2, v38, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v0, v37, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v25, 16, v35
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v31, 16, v34
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v29, 16, v33
-; GFX9-DS128-NEXT: v_bfe_i32 v24, v35, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v30, v34, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v28, v33, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v49, 16, v36
-; GFX9-DS128-NEXT: v_bfe_i32 v48, v36, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v36, 16, v43
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v34, 16, v42
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v55, 16, v41
-; GFX9-DS128-NEXT: v_bfe_i32 v35, v43, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v33, v42, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v54, v41, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v62, 16, v59
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v60, 16, v58
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v57
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v56
-; GFX9-DS128-NEXT: v_bfe_i32 v61, v59, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v59, v58, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v57, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v56, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v43, 16, v40
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v41, 16, v39
-; GFX9-DS128-NEXT: v_bfe_i32 v42, v40, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v40, v39, 0, 16
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[0:3] offset:224
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[40:43] offset:240
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[4:7] offset:192
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[59:62] offset:208
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[52:55] offset:160
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[33:36] offset:176
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[48:51] offset:128
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[44:47] offset:144
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[28:31] offset:96
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[24:27] offset:112
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[20:23] offset:64
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[16:19] offset:80
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[12:15] offset:32
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[8:11] offset:48
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v53, 16, v50
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v51, 16, v49
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v57, 16, v48
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v55, 16, v47
+; GFX9-DS128-NEXT: v_bfe_i32 v52, v50, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v50, v49, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v56, v48, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v54, v47, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 16, v61
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 16, v60
+; GFX9-DS128-NEXT: v_bfe_i32 v6, v61, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v4, v60, 0, 16
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[4:7] offset:240
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[8:11] offset:192
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[12:15] offset:208
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[54:57] offset:160
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[50:53] offset:176
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[43:46] offset:128
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[39:42] offset:144
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[32:35] offset:96
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[25:28] offset:112
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[20:23] offset:64
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[16:19] offset:80
+; GFX9-DS128-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:32
+; GFX9-DS128-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:32 ; 4-byte Folded Reload
+; GFX9-DS128-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:36 ; 4-byte Folded Reload
+; GFX9-DS128-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:40 ; 4-byte Folded Reload
+; GFX9-DS128-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:44 ; 4-byte Folded Reload
+; GFX9-DS128-NEXT: s_waitcnt vmcnt(0)
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:48
; GFX9-DS128-NEXT: buffer_load_dword v0, off, s[12:15], 0 offset:16 ; 4-byte Folded Reload
; GFX9-DS128-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:20 ; 4-byte Folded Reload
; GFX9-DS128-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:24 ; 4-byte Folded Reload
; GFX9-DS128-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:28 ; 4-byte Folded Reload
; GFX9-DS128-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[0:3]
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[0:3]
; GFX9-DS128-NEXT: buffer_load_dword v0, off, s[12:15], 0 ; 4-byte Folded Reload
; GFX9-DS128-NEXT: buffer_load_dword v1, off, s[12:15], 0 offset:4 ; 4-byte Folded Reload
; GFX9-DS128-NEXT: buffer_load_dword v2, off, s[12:15], 0 offset:8 ; 4-byte Folded Reload
; GFX9-DS128-NEXT: buffer_load_dword v3, off, s[12:15], 0 offset:12 ; 4-byte Folded Reload
; GFX9-DS128-NEXT: s_waitcnt vmcnt(0)
-; GFX9-DS128-NEXT: ds_write_b128 v32, v[0:3] offset:16
+; GFX9-DS128-NEXT: ds_write_b128 v24, v[0:3] offset:16
; GFX9-DS128-NEXT: s_endpgm
%load = load <64 x i16>, ptr addrspace(3) %in
%ext = sext <64 x i16> %load to <64 x i32>
@@ -5757,17 +5814,17 @@ define amdgpu_kernel void @local_sextload_v4i16_to_v4i64(ptr addrspace(3) %out,
; VI-NO-DS128-NEXT: v_mov_b32_e32 v0, s1
; VI-NO-DS128-NEXT: ds_read_b64 v[0:1], v0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; VI-NO-DS128-NEXT: v_bfe_i32 v1, v1, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v7, v4, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v3, v0, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v5, v2, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v7, v4, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v8, 31, v7
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v0, s0
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v4, 31, v3
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v8, 31, v7
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v0, s0
; VI-NO-DS128-NEXT: ds_write2_b64 v0, v[1:2], v[7:8] offset0:2 offset1:3
; VI-NO-DS128-NEXT: ds_write2_b64 v0, v[3:4], v[5:6] offset1:1
; VI-NO-DS128-NEXT: s_endpgm
@@ -5780,16 +5837,16 @@ define amdgpu_kernel void @local_sextload_v4i16_to_v4i64(ptr addrspace(3) %out,
; GFX9-NO-DS128-NEXT: ds_read_b64 v[0:1], v0
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v9, s0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v1
; GFX9-NO-DS128-NEXT: v_bfe_i32 v1, v1, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v7, v4, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v3, v0, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v5, v2, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v7, v4, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v2, 31, v1
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v8, 31, v7
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v4, 31, v3
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v8, 31, v7
; GFX9-NO-DS128-NEXT: ds_write2_b64 v9, v[1:2], v[7:8] offset0:2 offset1:3
; GFX9-NO-DS128-NEXT: ds_write2_b64 v9, v[3:4], v[5:6] offset1:1
; GFX9-NO-DS128-NEXT: s_endpgm
@@ -5848,16 +5905,16 @@ define amdgpu_kernel void @local_sextload_v4i16_to_v4i64(ptr addrspace(3) %out,
; VI-DS128-NEXT: ds_read_b64 v[1:2], v0
; VI-DS128-NEXT: v_mov_b32_e32 v8, s0
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; VI-DS128-NEXT: v_bfe_i32 v0, v2, 0, 16
; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v4, v1, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v6, v3, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:16
; VI-DS128-NEXT: ds_write_b128 v8, v[4:7]
; VI-DS128-NEXT: s_endpgm
@@ -5870,16 +5927,16 @@ define amdgpu_kernel void @local_sextload_v4i16_to_v4i64(ptr addrspace(3) %out,
; GFX9-DS128-NEXT: ds_read_b64 v[1:2], v0
; GFX9-DS128-NEXT: v_mov_b32_e32 v8, s0
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v1
; GFX9-DS128-NEXT: v_bfe_i32 v0, v2, 0, 16
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v1
-; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v4, v1, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v6, v3, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:16
; GFX9-DS128-NEXT: ds_write_b128 v8, v[4:7]
; GFX9-DS128-NEXT: s_endpgm
@@ -6170,26 +6227,26 @@ define amdgpu_kernel void @local_sextload_v8i16_to_v8i64(ptr addrspace(3) %out,
; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v0 offset1:1
; VI-NO-DS128-NEXT: v_mov_b32_e32 v16, s0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v3
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; VI-NO-DS128-NEXT: v_bfe_i32 v10, v10, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v14, v2, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v2, v3, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v3
; VI-NO-DS128-NEXT: v_bfe_i32 v4, v4, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v6, v5, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v8, v7, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; VI-NO-DS128-NEXT: v_bfe_i32 v10, v10, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v12, v0, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v0, v1, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-NO-DS128-NEXT: v_bfe_i32 v14, v2, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v2, v3, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-NO-DS128-NEXT: ds_write2_b64 v16, v[2:3], v[10:11] offset0:6 offset1:7
; VI-NO-DS128-NEXT: ds_write2_b64 v16, v[14:15], v[8:9] offset0:4 offset1:5
; VI-NO-DS128-NEXT: ds_write2_b64 v16, v[0:1], v[6:7] offset0:2 offset1:3
@@ -6204,26 +6261,26 @@ define amdgpu_kernel void @local_sextload_v8i16_to_v8i64(ptr addrspace(3) %out,
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v0 offset1:1
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v16, s0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v3
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v0
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v1
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v2
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v10, v9, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v14, v2, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v3, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v3
; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v4, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v5, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v8, v7, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v10, v9, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v12, v0, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v1, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v14, v2, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v3, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[2:3], v[10:11] offset0:6 offset1:7
; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[14:15], v[8:9] offset0:4 offset1:5
; GFX9-NO-DS128-NEXT: ds_write2_b64 v16, v[0:1], v[6:7] offset0:2 offset1:3
@@ -6327,24 +6384,24 @@ define amdgpu_kernel void @local_sextload_v8i16_to_v8i64(ptr addrspace(3) %out,
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-DS128-NEXT: v_bfe_i32 v4, v0, 0, 16
; VI-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; VI-DS128-NEXT: v_bfe_i32 v8, v1, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; VI-DS128-NEXT: v_bfe_i32 v6, v0, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v12, v2, 0, 16
; VI-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; VI-DS128-NEXT: v_bfe_i32 v8, v1, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; VI-DS128-NEXT: v_bfe_i32 v10, v1, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v14, v0, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v0, v3, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v10, v1, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; VI-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; VI-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; VI-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; VI-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-DS128-NEXT: ds_write_b128 v16, v[0:3] offset:48
; VI-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:32
; VI-DS128-NEXT: ds_write_b128 v16, v[8:11] offset:16
@@ -6361,24 +6418,24 @@ define amdgpu_kernel void @local_sextload_v8i16_to_v8i64(ptr addrspace(3) %out,
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-DS128-NEXT: v_bfe_i32 v4, v0, 0, 16
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v0
+; GFX9-DS128-NEXT: v_bfe_i32 v8, v1, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v1
; GFX9-DS128-NEXT: v_bfe_i32 v12, v2, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v6, v0, 0, 16
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v2
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX9-DS128-NEXT: v_bfe_i32 v8, v1, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v1
+; GFX9-DS128-NEXT: v_bfe_i32 v10, v1, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v14, v0, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v0, v3, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v10, v1, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX9-DS128-NEXT: ds_write_b128 v16, v[0:3] offset:48
; GFX9-DS128-NEXT: ds_write_b128 v16, v[12:15] offset:32
; GFX9-DS128-NEXT: ds_write_b128 v16, v[8:11] offset:16
@@ -6399,11 +6456,11 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i64(ptr addrspace(3) %out
; SI-NEXT: s_mov_b32 m0, -1
; SI-NEXT: ds_read2_b64 v[0:3], v4 offset0:2 offset1:3
; SI-NEXT: v_mov_b32_e32 v9, 0
-; SI-NEXT: ds_read2_b64 v[4:7], v4 offset1:1
; SI-NEXT: v_mov_b32_e32 v11, v9
; SI-NEXT: v_mov_b32_e32 v13, v9
; SI-NEXT: v_mov_b32_e32 v15, v9
; SI-NEXT: v_mov_b32_e32 v17, v9
+; SI-NEXT: ds_read2_b64 v[4:7], v4 offset1:1
; SI-NEXT: v_mov_b32_e32 v20, s0
; SI-NEXT: s_waitcnt lgkmcnt(1)
; SI-NEXT: v_lshrrev_b32_e32 v14, 16, v1
@@ -6743,53 +6800,53 @@ define amdgpu_kernel void @local_zextload_v16i16_to_v16i64(ptr addrspace(3) %out
; GFX9-DS128-LABEL: local_zextload_v16i16_to_v16i64:
; GFX9-DS128: ; %bb.0:
; GFX9-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-DS128-NEXT: v_mov_b32_e32 v25, 0
-; GFX9-DS128-NEXT: v_mov_b32_e32 v21, v25
-; GFX9-DS128-NEXT: v_mov_b32_e32 v23, v25
-; GFX9-DS128-NEXT: v_mov_b32_e32 v18, v25
+; GFX9-DS128-NEXT: v_mov_b32_e32 v26, 0
+; GFX9-DS128-NEXT: v_mov_b32_e32 v22, v26
+; GFX9-DS128-NEXT: v_mov_b32_e32 v24, v26
+; GFX9-DS128-NEXT: v_mov_b32_e32 v11, v26
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v4
-; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v4 offset:16
-; GFX9-DS128-NEXT: v_mov_b32_e32 v28, s0
-; GFX9-DS128-NEXT: v_mov_b32_e32 v15, v25
-; GFX9-DS128-NEXT: v_mov_b32_e32 v12, v25
+; GFX9-DS128-NEXT: v_mov_b32_e32 v5, s1
+; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v5
+; GFX9-DS128-NEXT: ds_read_b128 v[13:16], v5 offset:16
+; GFX9-DS128-NEXT: v_mov_b32_e32 v19, v26
+; GFX9-DS128-NEXT: v_mov_b32_e32 v8, v26
+; GFX9-DS128-NEXT: v_mov_b32_e32 v28, v26
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v2
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v12, 16, v2
+; GFX9-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v2
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v7
-; GFX9-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v7
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v6
-; GFX9-DS128-NEXT: v_and_b32_e32 v17, 0xffff, v6
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[20:23] offset:112
-; GFX9-DS128-NEXT: v_mov_b32_e32 v20, v25
-; GFX9-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v2
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[17:20] offset:96
-; GFX9-DS128-NEXT: v_mov_b32_e32 v17, v25
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v1
-; GFX9-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v1
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[14:17] offset:32
-; GFX9-DS128-NEXT: v_mov_b32_e32 v14, v25
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v0
-; GFX9-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v0
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v23, 16, v13
+; GFX9-DS128-NEXT: v_and_b32_e32 v21, 0xffff, v13
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v27, 16, v14
+; GFX9-DS128-NEXT: v_and_b32_e32 v25, 0xffff, v14
+; GFX9-DS128-NEXT: v_mov_b32_e32 v14, s0
+; GFX9-DS128-NEXT: v_mov_b32_e32 v13, v26
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v1
+; GFX9-DS128-NEXT: v_and_b32_e32 v7, 0xffff, v1
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v16
+; GFX9-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v16
+; GFX9-DS128-NEXT: ds_write_b128 v14, v[21:24] offset:64
+; GFX9-DS128-NEXT: v_mov_b32_e32 v21, v26
+; GFX9-DS128-NEXT: ds_write_b128 v14, v[10:13] offset:32
+; GFX9-DS128-NEXT: v_mov_b32_e32 v10, v26
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v0
+; GFX9-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v0
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
; GFX9-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v3
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX9-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v4
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v26, 16, v5
-; GFX9-DS128-NEXT: v_and_b32_e32 v24, 0xffff, v5
-; GFX9-DS128-NEXT: v_mov_b32_e32 v5, v25
-; GFX9-DS128-NEXT: v_mov_b32_e32 v7, v25
-; GFX9-DS128-NEXT: v_mov_b32_e32 v1, v25
-; GFX9-DS128-NEXT: v_mov_b32_e32 v3, v25
-; GFX9-DS128-NEXT: v_mov_b32_e32 v27, v25
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[11:14] offset:16
-; GFX9-DS128-NEXT: v_mov_b32_e32 v9, v25
-; GFX9-DS128-NEXT: v_mov_b32_e32 v11, v25
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[4:7] offset:64
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[0:3] offset:48
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[24:27] offset:80
-; GFX9-DS128-NEXT: ds_write_b128 v28, v[8:11]
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v15
+; GFX9-DS128-NEXT: v_and_b32_e32 v15, 0xffff, v15
+; GFX9-DS128-NEXT: ds_write_b128 v14, v[18:21] offset:112
+; GFX9-DS128-NEXT: v_mov_b32_e32 v16, v26
+; GFX9-DS128-NEXT: v_mov_b32_e32 v18, v26
+; GFX9-DS128-NEXT: v_mov_b32_e32 v1, v26
+; GFX9-DS128-NEXT: v_mov_b32_e32 v3, v26
+; GFX9-DS128-NEXT: ds_write_b128 v14, v[7:10] offset:16
+; GFX9-DS128-NEXT: v_mov_b32_e32 v5, v26
+; GFX9-DS128-NEXT: v_mov_b32_e32 v7, v26
+; GFX9-DS128-NEXT: ds_write_b128 v14, v[15:18] offset:96
+; GFX9-DS128-NEXT: ds_write_b128 v14, v[0:3] offset:48
+; GFX9-DS128-NEXT: ds_write_b128 v14, v[25:28] offset:80
+; GFX9-DS128-NEXT: ds_write_b128 v14, v[4:7]
; GFX9-DS128-NEXT: s_endpgm
%load = load <16 x i16>, ptr addrspace(3) %in
%ext = zext <16 x i16> %load to <16 x i64>
@@ -6865,7 +6922,7 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i64(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: ds_read2_b64 v[4:7], v4 offset0:2 offset1:3
; VI-NO-DS128-NEXT: v_mov_b32_e32 v19, s0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v3
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v4
; VI-NO-DS128-NEXT: v_bfe_i32 v14, v14, 0, 16
@@ -6888,28 +6945,28 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i64(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[14:15], v[4:5] offset0:12 offset1:13
; VI-NO-DS128-NEXT: v_bfe_i32 v14, v6, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v1
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v2
-; VI-NO-DS128-NEXT: v_bfe_i32 v4, v18, 0, 16
-; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[6:7], v[14:15] offset0:14 offset1:15
-; VI-NO-DS128-NEXT: v_bfe_i32 v6, v2, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v2, v3, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v3
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; VI-NO-DS128-NEXT: v_bfe_i32 v8, v8, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v10, v9, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v12, v11, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; VI-NO-DS128-NEXT: v_bfe_i32 v4, v18, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v16, v0, 0, 16
+; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[6:7], v[14:15] offset0:14 offset1:15
; VI-NO-DS128-NEXT: v_bfe_i32 v0, v1, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; VI-NO-DS128-NEXT: v_bfe_i32 v6, v2, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v2, v3, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[2:3], v[4:5] offset0:6 offset1:7
; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[6:7], v[12:13] offset0:4 offset1:5
; VI-NO-DS128-NEXT: ds_write2_b64 v19, v[0:1], v[10:11] offset0:2 offset1:3
@@ -6925,7 +6982,7 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i64(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v4 offset0:2 offset1:3
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v19, s0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v3
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v4
; GFX9-NO-DS128-NEXT: v_bfe_i32 v14, v14, 0, 16
@@ -6948,28 +7005,28 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i64(ptr addrspace(3) %out
; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[14:15], v[4:5] offset0:12 offset1:13
; GFX9-NO-DS128-NEXT: v_bfe_i32 v14, v6, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v0
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v1
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v11, 16, v2
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v18, 0, 16
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[6:7], v[14:15] offset0:14 offset1:15
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v2, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v3, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v3
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GFX9-NO-DS128-NEXT: v_bfe_i32 v8, v8, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v10, v9, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v12, v11, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v4, v18, 0, 16
; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v0, 0, 16
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[6:7], v[14:15] offset0:14 offset1:15
; GFX9-NO-DS128-NEXT: v_bfe_i32 v0, v1, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v6, v2, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v3, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[2:3], v[4:5] offset0:6 offset1:7
; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[6:7], v[12:13] offset0:4 offset1:5
; GFX9-NO-DS128-NEXT: ds_write2_b64 v19, v[0:1], v[10:11] offset0:2 offset1:3
@@ -7188,20 +7245,20 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i64(ptr addrspace(3) %out
; VI-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v5
; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
; VI-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:96
+; VI-DS128-NEXT: v_bfe_i32 v14, v5, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v14, v5, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; VI-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; VI-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
; VI-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; VI-DS128-NEXT: ds_write_b128 v8, v[4:7] offset:48
; VI-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:32
; VI-DS128-NEXT: ds_write_b128 v8, v[10:13] offset:16
@@ -7248,20 +7305,20 @@ define amdgpu_kernel void @local_sextload_v16i16_to_v16i64(ptr addrspace(3) %out
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v5
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
; GFX9-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:96
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v5, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
; GFX9-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v14, v5, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GFX9-DS128-NEXT: ds_write_b128 v8, v[4:7] offset:48
; GFX9-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:32
; GFX9-DS128-NEXT: ds_write_b128 v8, v[10:13] offset:16
@@ -7277,42 +7334,41 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; SI-LABEL: local_zextload_v32i16_to_v32i64:
; SI: ; %bb.0:
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
+; SI-NEXT: v_mov_b32_e32 v1, 0
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: v_mov_b32_e32 v0, s1
; SI-NEXT: s_mov_b32 m0, -1
; SI-NEXT: ds_read2_b64 v[2:5], v0 offset0:2 offset1:3
-; SI-NEXT: v_mov_b32_e32 v1, 0
-; SI-NEXT: ds_read2_b64 v[6:9], v0 offset1:1
; SI-NEXT: v_mov_b32_e32 v19, v1
; SI-NEXT: v_mov_b32_e32 v21, v1
+; SI-NEXT: ds_read2_b64 v[6:9], v0 offset0:4 offset1:5
+; SI-NEXT: ds_read2_b64 v[10:13], v0 offset0:6 offset1:7
; SI-NEXT: v_mov_b32_e32 v22, s0
-; SI-NEXT: s_waitcnt lgkmcnt(1)
+; SI-NEXT: s_waitcnt lgkmcnt(2)
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v5
; SI-NEXT: v_and_b32_e32 v20, 0xffff, v5
-; SI-NEXT: ds_read2_b64 v[10:13], v0 offset0:4 offset1:5
-; SI-NEXT: ds_read2_b64 v[14:17], v0 offset0:6 offset1:7
+; SI-NEXT: ds_read2_b64 v[14:17], v0 offset1:1
; SI-NEXT: ds_write2_b64 v22, v[20:21], v[18:19] offset0:14 offset1:15
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v3
; SI-NEXT: v_and_b32_e32 v20, 0xffff, v3
; SI-NEXT: ds_write2_b64 v22, v[20:21], v[18:19] offset0:10 offset1:11
-; SI-NEXT: s_waitcnt lgkmcnt(4)
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v9
-; SI-NEXT: v_and_b32_e32 v20, 0xffff, v9
-; SI-NEXT: ds_write2_b64 v22, v[20:21], v[18:19] offset0:6 offset1:7
-; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v7
-; SI-NEXT: v_and_b32_e32 v20, 0xffff, v7
-; SI-NEXT: ds_write2_b64 v22, v[20:21], v[18:19] offset0:2 offset1:3
-; SI-NEXT: s_waitcnt lgkmcnt(4)
+; SI-NEXT: s_waitcnt lgkmcnt(2)
; SI-NEXT: v_lshrrev_b32_e32 v18, 16, v17
; SI-NEXT: v_and_b32_e32 v20, 0xffff, v17
-; SI-NEXT: ds_write2_b64 v22, v[20:21], v[18:19] offset0:30 offset1:31
+; SI-NEXT: ds_write2_b64 v22, v[20:21], v[18:19] offset0:6 offset1:7
; SI-NEXT: v_mov_b32_e32 v18, v1
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v15
; SI-NEXT: v_mov_b32_e32 v20, v1
; SI-NEXT: v_and_b32_e32 v19, 0xffff, v15
-; SI-NEXT: ds_write2_b64 v22, v[19:20], v[17:18] offset0:26 offset1:27
+; SI-NEXT: ds_write2_b64 v22, v[19:20], v[17:18] offset0:2 offset1:3
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v13
; SI-NEXT: v_and_b32_e32 v19, 0xffff, v13
+; SI-NEXT: ds_write2_b64 v22, v[19:20], v[17:18] offset0:30 offset1:31
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v11
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v11
+; SI-NEXT: ds_write2_b64 v22, v[19:20], v[17:18] offset0:26 offset1:27
+; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v9
+; SI-NEXT: v_and_b32_e32 v19, 0xffff, v9
; SI-NEXT: ds_write2_b64 v22, v[19:20], v[17:18] offset0:22 offset1:23
; SI-NEXT: v_lshrrev_b32_e32 v17, 16, v4
; SI-NEXT: v_mov_b32_e32 v5, v1
@@ -7322,194 +7378,199 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; SI-NEXT: v_and_b32_e32 v17, 0xffff, v2
; SI-NEXT: v_mov_b32_e32 v4, v1
; SI-NEXT: ds_write2_b64 v22, v[17:18], v[3:4] offset0:8 offset1:9
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v8
-; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v6
-; SI-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; SI-NEXT: v_and_b32_e32 v8, 0xffff, v8
-; SI-NEXT: v_mov_b32_e32 v9, v1
-; SI-NEXT: v_mov_b32_e32 v7, v1
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v16
+; SI-NEXT: v_lshrrev_b32_e32 v4, 16, v14
+; SI-NEXT: v_and_b32_e32 v13, 0xffff, v14
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v16
+; SI-NEXT: v_mov_b32_e32 v16, v1
+; SI-NEXT: v_mov_b32_e32 v14, v1
; SI-NEXT: v_mov_b32_e32 v3, v1
-; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; SI-NEXT: ds_write2_b64 v22, v[8:9], v[2:3] offset0:4 offset1:5
-; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v12
-; SI-NEXT: v_lshrrev_b32_e32 v8, 16, v10
-; SI-NEXT: ds_write2_b64 v22, v[6:7], v[4:5] offset1:1
-; SI-NEXT: v_and_b32_e32 v4, 0xffff, v10
-; SI-NEXT: v_and_b32_e32 v5, 0xffff, v11
-; SI-NEXT: v_and_b32_e32 v10, 0xffff, v12
-; SI-NEXT: v_lshrrev_b32_e32 v12, 16, v16
-; SI-NEXT: v_lshrrev_b32_e32 v15, 16, v14
-; SI-NEXT: v_and_b32_e32 v17, 0xffff, v14
-; SI-NEXT: v_and_b32_e32 v19, 0xffff, v16
+; SI-NEXT: v_lshrrev_b32_e32 v0, 16, v7
+; SI-NEXT: ds_write2_b64 v22, v[15:16], v[2:3] offset0:4 offset1:5
+; SI-NEXT: v_lshrrev_b32_e32 v2, 16, v8
+; SI-NEXT: v_lshrrev_b32_e32 v9, 16, v6
+; SI-NEXT: ds_write2_b64 v22, v[13:14], v[4:5] offset1:1
+; SI-NEXT: v_and_b32_e32 v4, 0xffff, v6
+; SI-NEXT: v_and_b32_e32 v5, 0xffff, v7
+; SI-NEXT: v_and_b32_e32 v7, 0xffff, v8
+; SI-NEXT: v_lshrrev_b32_e32 v11, 16, v12
+; SI-NEXT: v_lshrrev_b32_e32 v13, 16, v10
+; SI-NEXT: v_and_b32_e32 v15, 0xffff, v10
+; SI-NEXT: v_and_b32_e32 v17, 0xffff, v12
; SI-NEXT: v_mov_b32_e32 v6, v1
; SI-NEXT: ds_write2_b64 v22, v[5:6], v[0:1] offset0:18 offset1:19
-; SI-NEXT: v_mov_b32_e32 v11, v1
+; SI-NEXT: v_mov_b32_e32 v8, v1
; SI-NEXT: v_mov_b32_e32 v5, v1
-; SI-NEXT: v_mov_b32_e32 v13, v1
-; SI-NEXT: v_mov_b32_e32 v16, v1
-; SI-NEXT: ds_write2_b64 v22, v[19:20], v[12:13] offset0:28 offset1:29
-; SI-NEXT: ds_write2_b64 v22, v[17:18], v[15:16] offset0:24 offset1:25
-; SI-NEXT: ds_write2_b64 v22, v[10:11], v[2:3] offset0:20 offset1:21
-; SI-NEXT: ds_write2_b64 v22, v[4:5], v[8:9] offset0:16 offset1:17
+; SI-NEXT: v_mov_b32_e32 v12, v1
+; SI-NEXT: v_mov_b32_e32 v10, v1
+; SI-NEXT: ds_write2_b64 v22, v[17:18], v[11:12] offset0:28 offset1:29
+; SI-NEXT: ds_write2_b64 v22, v[15:16], v[13:14] offset0:24 offset1:25
+; SI-NEXT: ds_write2_b64 v22, v[7:8], v[2:3] offset0:20 offset1:21
+; SI-NEXT: ds_write2_b64 v22, v[4:5], v[9:10] offset0:16 offset1:17
; SI-NEXT: s_endpgm
;
; VI-NO-DS128-LABEL: local_zextload_v32i16_to_v32i64:
; VI-NO-DS128: ; %bb.0:
; VI-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-NO-DS128-NEXT: s_mov_b32 m0, -1
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v5, 0
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v19, v5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v21, v5
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v1, 0
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v19, v1
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v21, v1
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
-; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset0:6 offset1:7
-; VI-NO-DS128-NEXT: ds_read2_b64 v[6:9], v4 offset0:4 offset1:5
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v0, s1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[2:5], v0 offset0:6 offset1:7
+; VI-NO-DS128-NEXT: ds_read2_b64 v[6:9], v0 offset0:4 offset1:5
; VI-NO-DS128-NEXT: v_mov_b32_e32 v22, s0
-; VI-NO-DS128-NEXT: ds_read2_b64 v[10:13], v4 offset0:2 offset1:3
-; VI-NO-DS128-NEXT: ds_read2_b64 v[14:17], v4 offset1:1
+; VI-NO-DS128-NEXT: ds_read2_b64 v[10:13], v0 offset0:2 offset1:3
+; VI-NO-DS128-NEXT: ds_read2_b64 v[14:17], v0 offset1:1
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v2
-; VI-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v2
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v4
+; VI-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v4
; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[20:21] offset0:28 offset1:29
-; VI-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v1
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v2, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[1:2] offset0:26 offset1:27
-; VI-NO-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v0
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[1:2], v[18:19] offset0:24 offset1:25
+; VI-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v3
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, v1
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[3:4] offset0:26 offset1:27
+; VI-NO-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v2
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v2
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[3:4], v[18:19] offset0:24 offset1:25
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(5)
-; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v1, v5
+; VI-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v9
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v3, v1
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v9
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[18:19] offset0:22 offset1:23
-; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v8
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[2:3], v[18:19] offset0:22 offset1:23
+; VI-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v8
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v9, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[8:9] offset0:20 offset1:21
-; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v7
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v9, v1
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[2:3], v[8:9] offset0:20 offset1:21
+; VI-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v7
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v8, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[7:8] offset0:18 offset1:19
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v6
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v8, v1
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[2:3], v[7:8] offset0:18 offset1:19
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v6
; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v7, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:16 offset1:17
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v7, v1
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[2:3] offset0:16 offset1:17
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(8)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v13
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v13
; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v13
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:14 offset1:15
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v12
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[2:3] offset0:14 offset1:15
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v12
; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v12
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:12 offset1:13
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v11
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[2:3] offset0:12 offset1:13
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v11
; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v11
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:10 offset1:11
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[2:3] offset0:10 offset1:11
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(10)
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v14
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v14
; VI-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v14
-; VI-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v3
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v14, v5
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v10
+; VI-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v5
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v14, v1
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v10
; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v10
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v17
; VI-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v17
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[4:5], v[13:14] offset0:30 offset1:31
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v13, v5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v11, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[1:2] offset0:8 offset1:9
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v16
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[13:14] offset0:30 offset1:31
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v13, v1
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v11, v1
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[3:4] offset0:8 offset1:9
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v16
; VI-NO-DS128-NEXT: v_and_b32_e32 v9, 0xffff, v16
; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[12:13], v[10:11] offset0:6 offset1:7
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v10, v5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v3, v5
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v15
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v10, v1
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v5, v1
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v15
; VI-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v15
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[9:10], v[2:3] offset0:4 offset1:5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v2, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[1:2] offset0:2 offset1:3
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v9, v5
-; VI-NO-DS128-NEXT: v_mov_b32_e32 v1, v5
-; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[8:9], v[0:1] offset1:1
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[9:10], v[4:5] offset0:4 offset1:5
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, v1
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[3:4] offset0:2 offset1:3
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v9, v1
+; VI-NO-DS128-NEXT: v_mov_b32_e32 v3, v1
+; VI-NO-DS128-NEXT: ds_write2_b64 v22, v[8:9], v[2:3] offset1:1
; VI-NO-DS128-NEXT: s_endpgm
;
; GFX9-NO-DS128-LABEL: local_zextload_v32i16_to_v32i64:
; GFX9-NO-DS128: ; %bb.0:
; GFX9-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v5, 0
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v19, v5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v21, v5
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v1, 0
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v19, v1
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v21, v1
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[6:9], v4 offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v0, s1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[2:5], v0 offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[14:17], v0 offset0:4 offset1:5
; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v22, s0
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[10:13], v4 offset1:1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[14:17], v4 offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(2)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v2
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v2
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[6:9], v0 offset1:1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[10:13], v0 offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(3)
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v4
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v4
; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[20:21] offset0:28 offset1:29
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v1
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v1
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v2, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[1:2] offset0:26 offset1:27
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v0
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v0
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[1:2], v[18:19] offset0:24 offset1:25
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v9
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v9
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[18:19] offset0:22 offset1:23
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v8
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v8
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v9, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[8:9] offset0:20 offset1:21
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v7
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v7
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v8, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[7:8] offset0:18 offset1:19
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v6
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v7, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:16 offset1:17
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v18, 0xffff, v3
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v4, v1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[18:19], v[3:4] offset0:26 offset1:27
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v3, 0xffff, v2
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v18, 16, v2
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[3:4], v[18:19] offset0:24 offset1:25
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(5)
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v17
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v3, v1
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v17
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v18, v1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[2:3], v[17:18] offset0:22 offset1:23
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v16
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v17, v1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[2:3], v[16:17] offset0:20 offset1:21
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v2, 0xffff, v15
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v15, 16, v15
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v16, v1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[2:3], v[15:16] offset0:18 offset1:19
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v14
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v14
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v15, v1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[14:15], v[2:3] offset0:16 offset1:17
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(7)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v17
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v17
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:14 offset1:15
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v16
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v16
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:12 offset1:13
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v15
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v15
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[0:1] offset0:10 offset1:11
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v14
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v14
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v10
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[1:2] offset0:8 offset1:9
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v13
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v13, 0xffff, v13
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v14, v1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[13:14], v[2:3] offset0:14 offset1:15
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v12
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v9, 0xffff, v12
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v13
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v13
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v3
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v3
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v14, v5
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v11
-; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v11
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[4:5], v[13:14] offset0:30 offset1:31
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v13, v5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v11, v5
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v12
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v13, v1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[12:13], v[2:3] offset0:12 offset1:13
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v11
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v11
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v12, v1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[11:12], v[2:3] offset0:10 offset1:11
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v10
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v10, 0xffff, v10
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v11, v1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[10:11], v[3:4] offset0:8 offset1:9
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v8
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v8
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v9
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v12, 0xffff, v9
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v5
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v5
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v9, v1
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v5, v1
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v7
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v7, 0xffff, v7
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[8:9], v[4:5] offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v8, v1
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v4, v1
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v6
+; GFX9-NO-DS128-NEXT: v_and_b32_e32 v6, 0xffff, v6
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[0:1], v[13:14] offset0:30 offset1:31
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v13, v1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[7:8], v[3:4] offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v7, v1
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v3, v1
; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[12:13], v[10:11] offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v10, v5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v3, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[9:10], v[2:3] offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v2, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[1:2] offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v9, v5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v1, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[8:9], v[0:1] offset1:1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v22, v[6:7], v[2:3] offset1:1
; GFX9-NO-DS128-NEXT: s_endpgm
;
; EG-LABEL: local_zextload_v32i16_to_v32i64:
@@ -7811,193 +7872,193 @@ define amdgpu_kernel void @local_zextload_v32i16_to_v32i64(ptr addrspace(3) %out
; VI-DS128: ; %bb.0:
; VI-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; VI-DS128-NEXT: s_mov_b32 m0, -1
+; VI-DS128-NEXT: v_mov_b32_e32 v52, 0
+; VI-DS128-NEXT: v_mov_b32_e32 v48, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v50, v52
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_mov_b32_e32 v1, s1
-; VI-DS128-NEXT: ds_read_b128 v[3:6], v1
-; VI-DS128-NEXT: ds_read_b128 v[7:10], v1 offset:16
-; VI-DS128-NEXT: v_mov_b32_e32 v52, s0
+; VI-DS128-NEXT: v_mov_b32_e32 v9, s1
+; VI-DS128-NEXT: ds_read_b128 v[4:7], v9 offset:16
+; VI-DS128-NEXT: ds_read_b128 v[0:3], v9
+; VI-DS128-NEXT: v_mov_b32_e32 v45, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v42, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v39, v52
; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v6
-; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v8
-; VI-DS128-NEXT: v_and_b32_e32 v17, 0xffff, v8
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v7
-; VI-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v7
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v10
-; VI-DS128-NEXT: v_and_b32_e32 v23, 0xffff, v10
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v28, 16, v9
-; VI-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v9
-; VI-DS128-NEXT: ds_read_b128 v[7:10], v1 offset:32
-; VI-DS128-NEXT: ds_read_b128 v[29:32], v1 offset:48
-; VI-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v6
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; VI-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v5
+; VI-DS128-NEXT: v_and_b32_e32 v17, 0xffff, v5
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v4
+; VI-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v4
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v7
+; VI-DS128-NEXT: v_and_b32_e32 v23, 0xffff, v7
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v28, 16, v6
+; VI-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v6
+; VI-DS128-NEXT: ds_read_b128 v[4:7], v9 offset:32
; VI-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v38, 16, v7
-; VI-DS128-NEXT: v_and_b32_e32 v36, 0xffff, v7
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v44, 16, v9
-; VI-DS128-NEXT: v_and_b32_e32 v42, 0xffff, v9
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v0
+; VI-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v0
+; VI-DS128-NEXT: v_mov_b32_e32 v0, s0
+; VI-DS128-NEXT: v_mov_b32_e32 v36, v52
+; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v5
+; VI-DS128-NEXT: v_and_b32_e32 v29, 0xffff, v5
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v34, 16, v4
+; VI-DS128-NEXT: v_and_b32_e32 v32, 0xffff, v4
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v37, 16, v7
+; VI-DS128-NEXT: v_and_b32_e32 v35, 0xffff, v7
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v40, 16, v6
+; VI-DS128-NEXT: v_and_b32_e32 v38, 0xffff, v6
+; VI-DS128-NEXT: ds_read_b128 v[4:7], v9 offset:48
+; VI-DS128-NEXT: v_mov_b32_e32 v33, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v30, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v27, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v24, v52
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v30
-; VI-DS128-NEXT: v_and_b32_e32 v7, 0xffff, v30
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v50, 16, v32
-; VI-DS128-NEXT: v_and_b32_e32 v48, 0xffff, v32
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v32, 16, v31
-; VI-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v31, 0
-; VI-DS128-NEXT: v_mov_b32_e32 v49, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v51, v31
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v47, 16, v29
-; VI-DS128-NEXT: v_and_b32_e32 v45, 0xffff, v29
-; VI-DS128-NEXT: ds_write_b128 v52, v[48:51] offset:240
-; VI-DS128-NEXT: v_mov_b32_e32 v46, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v48, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v27, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v29, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[45:48] offset:192
-; VI-DS128-NEXT: v_mov_b32_e32 v43, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v45, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[26:29] offset:96
-; VI-DS128-NEXT: v_mov_b32_e32 v24, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v26, v31
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v41, 16, v10
-; VI-DS128-NEXT: v_and_b32_e32 v39, 0xffff, v10
-; VI-DS128-NEXT: ds_write_b128 v52, v[42:45] offset:160
-; VI-DS128-NEXT: v_mov_b32_e32 v40, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v42, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[23:26] offset:112
-; VI-DS128-NEXT: v_mov_b32_e32 v21, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v23, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[39:42] offset:176
-; VI-DS128-NEXT: v_mov_b32_e32 v37, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v39, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[20:23] offset:64
-; VI-DS128-NEXT: v_mov_b32_e32 v18, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v20, v31
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v35, 16, v8
-; VI-DS128-NEXT: v_and_b32_e32 v33, 0xffff, v8
-; VI-DS128-NEXT: v_mov_b32_e32 v8, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v10, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[36:39] offset:128
-; VI-DS128-NEXT: v_mov_b32_e32 v34, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v36, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[17:20] offset:80
-; VI-DS128-NEXT: v_mov_b32_e32 v15, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v17, v31
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v3
-; VI-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v3
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; VI-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v5
-; VI-DS128-NEXT: ds_write_b128 v52, v[7:10] offset:208
-; VI-DS128-NEXT: ds_write_b128 v52, v[33:36] offset:144
-; VI-DS128-NEXT: v_mov_b32_e32 v5, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v7, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v33, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[14:17] offset:48
-; VI-DS128-NEXT: v_mov_b32_e32 v12, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v14, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v1, v31
-; VI-DS128-NEXT: v_mov_b32_e32 v3, v31
-; VI-DS128-NEXT: ds_write_b128 v52, v[4:7] offset:32
-; VI-DS128-NEXT: ds_write_b128 v52, v[30:33] offset:224
-; VI-DS128-NEXT: ds_write_b128 v52, v[11:14]
-; VI-DS128-NEXT: ds_write_b128 v52, v[0:3] offset:16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v49, 16, v7
+; VI-DS128-NEXT: v_and_b32_e32 v47, 0xffff, v7
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v46, 16, v4
+; VI-DS128-NEXT: v_and_b32_e32 v44, 0xffff, v4
+; VI-DS128-NEXT: ds_write_b128 v0, v[47:50] offset:240
+; VI-DS128-NEXT: v_mov_b32_e32 v47, v52
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v43, 16, v5
+; VI-DS128-NEXT: v_and_b32_e32 v41, 0xffff, v5
+; VI-DS128-NEXT: ds_write_b128 v0, v[44:47] offset:192
+; VI-DS128-NEXT: v_mov_b32_e32 v44, v52
+; VI-DS128-NEXT: ds_write_b128 v0, v[41:44] offset:208
+; VI-DS128-NEXT: v_mov_b32_e32 v41, v52
+; VI-DS128-NEXT: ds_write_b128 v0, v[38:41] offset:160
+; VI-DS128-NEXT: v_mov_b32_e32 v38, v52
+; VI-DS128-NEXT: ds_write_b128 v0, v[35:38] offset:176
+; VI-DS128-NEXT: v_mov_b32_e32 v35, v52
+; VI-DS128-NEXT: ds_write_b128 v0, v[32:35] offset:128
+; VI-DS128-NEXT: v_mov_b32_e32 v32, v52
+; VI-DS128-NEXT: ds_write_b128 v0, v[29:32] offset:144
+; VI-DS128-NEXT: v_mov_b32_e32 v29, v52
+; VI-DS128-NEXT: ds_write_b128 v0, v[26:29] offset:96
+; VI-DS128-NEXT: v_mov_b32_e32 v26, v52
+; VI-DS128-NEXT: ds_write_b128 v0, v[23:26] offset:112
+; VI-DS128-NEXT: v_mov_b32_e32 v21, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v23, v52
+; VI-DS128-NEXT: ds_write_b128 v0, v[20:23] offset:64
+; VI-DS128-NEXT: v_mov_b32_e32 v18, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v20, v52
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v3
+; VI-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v3
+; VI-DS128-NEXT: ds_write_b128 v0, v[17:20] offset:80
+; VI-DS128-NEXT: v_mov_b32_e32 v15, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v17, v52
+; VI-DS128-NEXT: ds_write_b128 v0, v[14:17] offset:48
+; VI-DS128-NEXT: v_mov_b32_e32 v12, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v14, v52
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; VI-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v1
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; VI-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v53, 16, v6
+; VI-DS128-NEXT: v_and_b32_e32 v51, 0xffff, v6
+; VI-DS128-NEXT: v_mov_b32_e32 v2, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v4, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v54, v52
+; VI-DS128-NEXT: ds_write_b128 v0, v[11:14]
+; VI-DS128-NEXT: v_mov_b32_e32 v9, v52
+; VI-DS128-NEXT: v_mov_b32_e32 v11, v52
+; VI-DS128-NEXT: ds_write_b128 v0, v[1:4] offset:32
+; VI-DS128-NEXT: ds_write_b128 v0, v[51:54] offset:224
+; VI-DS128-NEXT: ds_write_b128 v0, v[8:11] offset:16
; VI-DS128-NEXT: s_endpgm
;
; GFX9-DS128-LABEL: local_zextload_v32i16_to_v32i64:
; GFX9-DS128: ; %bb.0:
; GFX9-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
+; GFX9-DS128-NEXT: v_mov_b32_e32 v52, 0
+; GFX9-DS128-NEXT: v_mov_b32_e32 v48, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v50, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v45, v52
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_mov_b32_e32 v1, s1
-; GFX9-DS128-NEXT: ds_read_b128 v[3:6], v1
-; GFX9-DS128-NEXT: ds_read_b128 v[7:10], v1 offset:16
-; GFX9-DS128-NEXT: v_mov_b32_e32 v52, s0
+; GFX9-DS128-NEXT: v_mov_b32_e32 v9, s1
+; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v9 offset:16
+; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v9
+; GFX9-DS128-NEXT: v_mov_b32_e32 v42, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v39, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v36, v52
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v6
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v8
-; GFX9-DS128-NEXT: v_and_b32_e32 v17, 0xffff, v8
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v7
-; GFX9-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v7
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v10
-; GFX9-DS128-NEXT: v_and_b32_e32 v23, 0xffff, v10
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v28, 16, v9
-; GFX9-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v9
-; GFX9-DS128-NEXT: ds_read_b128 v[7:10], v1 offset:32
-; GFX9-DS128-NEXT: ds_read_b128 v[29:32], v1 offset:48
-; GFX9-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v6
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v4
-; GFX9-DS128-NEXT: v_and_b32_e32 v0, 0xffff, v4
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v19, 16, v5
+; GFX9-DS128-NEXT: v_and_b32_e32 v17, 0xffff, v5
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v22, 16, v4
+; GFX9-DS128-NEXT: v_and_b32_e32 v20, 0xffff, v4
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v25, 16, v7
+; GFX9-DS128-NEXT: v_and_b32_e32 v23, 0xffff, v7
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v28, 16, v6
+; GFX9-DS128-NEXT: v_and_b32_e32 v26, 0xffff, v6
+; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v9 offset:32
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v38, 16, v7
-; GFX9-DS128-NEXT: v_and_b32_e32 v36, 0xffff, v7
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v44, 16, v9
-; GFX9-DS128-NEXT: v_and_b32_e32 v42, 0xffff, v9
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v0
+; GFX9-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v0
+; GFX9-DS128-NEXT: v_mov_b32_e32 v0, s0
+; GFX9-DS128-NEXT: v_mov_b32_e32 v33, v52
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v9, 16, v30
-; GFX9-DS128-NEXT: v_and_b32_e32 v7, 0xffff, v30
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v50, 16, v32
-; GFX9-DS128-NEXT: v_and_b32_e32 v48, 0xffff, v32
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v32, 16, v31
-; GFX9-DS128-NEXT: v_and_b32_e32 v30, 0xffff, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v31, 0
-; GFX9-DS128-NEXT: v_mov_b32_e32 v49, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v51, v31
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v47, 16, v29
-; GFX9-DS128-NEXT: v_and_b32_e32 v45, 0xffff, v29
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[48:51] offset:240
-; GFX9-DS128-NEXT: v_mov_b32_e32 v46, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v48, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v27, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v29, v31
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[45:48] offset:192
-; GFX9-DS128-NEXT: v_mov_b32_e32 v43, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v45, v31
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[26:29] offset:96
-; GFX9-DS128-NEXT: v_mov_b32_e32 v24, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v26, v31
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v41, 16, v10
-; GFX9-DS128-NEXT: v_and_b32_e32 v39, 0xffff, v10
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[42:45] offset:160
-; GFX9-DS128-NEXT: v_mov_b32_e32 v40, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v42, v31
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[23:26] offset:112
-; GFX9-DS128-NEXT: v_mov_b32_e32 v21, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v23, v31
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[39:42] offset:176
-; GFX9-DS128-NEXT: v_mov_b32_e32 v37, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v39, v31
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[20:23] offset:64
-; GFX9-DS128-NEXT: v_mov_b32_e32 v18, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v20, v31
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v35, 16, v8
-; GFX9-DS128-NEXT: v_and_b32_e32 v33, 0xffff, v8
-; GFX9-DS128-NEXT: v_mov_b32_e32 v8, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v10, v31
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[36:39] offset:128
-; GFX9-DS128-NEXT: v_mov_b32_e32 v34, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v36, v31
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[17:20] offset:80
-; GFX9-DS128-NEXT: v_mov_b32_e32 v15, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v17, v31
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v3
-; GFX9-DS128-NEXT: v_and_b32_e32 v11, 0xffff, v3
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX9-DS128-NEXT: v_and_b32_e32 v4, 0xffff, v5
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[7:10] offset:208
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[33:36] offset:144
-; GFX9-DS128-NEXT: v_mov_b32_e32 v5, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v7, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v33, v31
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[14:17] offset:48
-; GFX9-DS128-NEXT: v_mov_b32_e32 v12, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v14, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v1, v31
-; GFX9-DS128-NEXT: v_mov_b32_e32 v3, v31
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[4:7] offset:32
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[30:33] offset:224
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[11:14]
-; GFX9-DS128-NEXT: ds_write_b128 v52, v[0:3] offset:16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v31, 16, v5
+; GFX9-DS128-NEXT: v_and_b32_e32 v29, 0xffff, v5
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v34, 16, v4
+; GFX9-DS128-NEXT: v_and_b32_e32 v32, 0xffff, v4
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v37, 16, v7
+; GFX9-DS128-NEXT: v_and_b32_e32 v35, 0xffff, v7
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v40, 16, v6
+; GFX9-DS128-NEXT: v_and_b32_e32 v38, 0xffff, v6
+; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v9 offset:48
+; GFX9-DS128-NEXT: v_mov_b32_e32 v30, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v27, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v24, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v21, v52
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v49, 16, v7
+; GFX9-DS128-NEXT: v_and_b32_e32 v47, 0xffff, v7
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v46, 16, v4
+; GFX9-DS128-NEXT: v_and_b32_e32 v44, 0xffff, v4
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[47:50] offset:240
+; GFX9-DS128-NEXT: v_mov_b32_e32 v47, v52
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v43, 16, v5
+; GFX9-DS128-NEXT: v_and_b32_e32 v41, 0xffff, v5
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[44:47] offset:192
+; GFX9-DS128-NEXT: v_mov_b32_e32 v44, v52
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[41:44] offset:208
+; GFX9-DS128-NEXT: v_mov_b32_e32 v41, v52
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[38:41] offset:160
+; GFX9-DS128-NEXT: v_mov_b32_e32 v38, v52
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[35:38] offset:176
+; GFX9-DS128-NEXT: v_mov_b32_e32 v35, v52
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[32:35] offset:128
+; GFX9-DS128-NEXT: v_mov_b32_e32 v32, v52
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[29:32] offset:144
+; GFX9-DS128-NEXT: v_mov_b32_e32 v29, v52
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[26:29] offset:96
+; GFX9-DS128-NEXT: v_mov_b32_e32 v26, v52
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[23:26] offset:112
+; GFX9-DS128-NEXT: v_mov_b32_e32 v23, v52
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[20:23] offset:64
+; GFX9-DS128-NEXT: v_mov_b32_e32 v18, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v20, v52
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v3
+; GFX9-DS128-NEXT: v_and_b32_e32 v14, 0xffff, v3
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[17:20] offset:80
+; GFX9-DS128-NEXT: v_mov_b32_e32 v15, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v17, v52
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[14:17] offset:48
+; GFX9-DS128-NEXT: v_mov_b32_e32 v12, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v14, v52
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v10, 16, v1
+; GFX9-DS128-NEXT: v_and_b32_e32 v8, 0xffff, v1
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
+; GFX9-DS128-NEXT: v_and_b32_e32 v1, 0xffff, v2
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v53, 16, v6
+; GFX9-DS128-NEXT: v_and_b32_e32 v51, 0xffff, v6
+; GFX9-DS128-NEXT: v_mov_b32_e32 v2, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v4, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v54, v52
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[11:14]
+; GFX9-DS128-NEXT: v_mov_b32_e32 v9, v52
+; GFX9-DS128-NEXT: v_mov_b32_e32 v11, v52
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[1:4] offset:32
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[51:54] offset:224
+; GFX9-DS128-NEXT: ds_write_b128 v0, v[8:11] offset:16
; GFX9-DS128-NEXT: s_endpgm
%load = load <32 x i16>, ptr addrspace(3) %in
%ext = zext <32 x i16> %load to <32 x i64>
@@ -8118,15 +8179,15 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-NO-DS128-NEXT: v_mov_b32_e32 v4, s1
; VI-NO-DS128-NEXT: ds_read2_b64 v[0:3], v4 offset0:6 offset1:7
-; VI-NO-DS128-NEXT: ds_read2_b64 v[12:15], v4 offset0:4 offset1:5
; VI-NO-DS128-NEXT: v_mov_b32_e32 v11, s0
-; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
+; VI-NO-DS128-NEXT: ds_read2_b64 v[12:15], v4 offset0:4 offset1:5
+; VI-NO-DS128-NEXT: ds_read2_b64 v[7:10], v4 offset0:2 offset1:3
+; VI-NO-DS128-NEXT: s_waitcnt lgkmcnt(2)
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v3
; VI-NO-DS128-NEXT: v_bfe_i32 v16, v3, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v18, v5, 0, 16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; VI-NO-DS128-NEXT: ds_read2_b64 v[7:10], v4 offset0:2 offset1:3
; VI-NO-DS128-NEXT: ds_read2_b64 v[3:6], v4 offset1:1
; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[16:17], v[18:19] offset0:30 offset1:31
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v2
@@ -8198,25 +8259,25 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[17:18], v[9:10] offset0:10 offset1:11
; VI-NO-DS128-NEXT: v_bfe_i32 v17, v0, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v19, v7, 0, 16
+; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v3
; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v12, 16, v6
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
-; VI-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v3
+; VI-NO-DS128-NEXT: v_bfe_i32 v1, v1, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v8, v12, 0, 16
+; VI-NO-DS128-NEXT: v_bfe_i32 v21, v3, 0, 16
; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[19:20], v[17:18] offset0:8 offset1:9
+; VI-NO-DS128-NEXT: v_bfe_i32 v3, v4, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v17, v5, 0, 16
; VI-NO-DS128-NEXT: v_bfe_i32 v5, v6, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v1, v1, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; VI-NO-DS128-NEXT: v_bfe_i32 v21, v3, 0, 16
-; VI-NO-DS128-NEXT: v_bfe_i32 v3, v4, 0, 16
-; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v2, 31, v1
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v22, 31, v21
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v4, 31, v3
; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; VI-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[5:6], v[8:9] offset0:6 offset1:7
; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[17:18], v[15:16] offset0:4 offset1:5
; VI-NO-DS128-NEXT: ds_write2_b64 v11, v[3:4], v[13:14] offset0:2 offset1:3
@@ -8227,111 +8288,110 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-NO-DS128: ; %bb.0:
; GFX9-NO-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v11, s1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v11 offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v11 offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v15, s0
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(1)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v8, 16, v7
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v7, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v8, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[7:10], v11 offset1:1
-; GFX9-NO-DS128-NEXT: ds_read2_b64 v[11:14], v11 offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[18:19] offset0:30 offset1:31
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v6
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v16, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v6, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:28 offset1:29
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v6, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v5, v5, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[5:6], v[16:17] offset0:26 offset1:27
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v4
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v17, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(5)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v3
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[18:19] offset0:24 offset1:25
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v4, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[3:4], v[16:17] offset0:22 offset1:23
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v13, s1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[0:3], v13 offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: v_mov_b32_e32 v12, s0
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[4:7], v13 offset1:1
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[8:11], v13 offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(2)
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v3
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v17, v3, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v19, v14, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v2
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v2, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v4, 31, v3
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-NO-DS128-NEXT: ds_read2_b64 v[13:16], v13 offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[17:18], v[19:20] offset0:30 offset1:31
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v17, v3, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[2:3], v[17:18] offset0:28 offset1:29
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[3:4] offset0:20 offset1:21
; GFX9-NO-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v1, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v17, v1, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[17:18], v[2:3] offset0:26 offset1:27
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v0
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v17, v0, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v19, v3, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(3)
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v16
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[17:18], v[19:20] offset0:24 offset1:25
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v17, v3, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v19, v16, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v15
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[19:20], v[17:18] offset0:22 offset1:23
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v3, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v15, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[2:3] offset0:18 offset1:19
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v17, 16, v0
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v14
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[18:19], v[16:17] offset0:20 offset1:21
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v15, v3, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v17, v14, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v14, v0, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v6
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[17:18], v[15:16] offset0:18 offset1:19
; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v0, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v17, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v13
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v13, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v20, v0, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v21, 31, v20
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[18:19] offset0:16 offset1:17
-; GFX9-NO-DS128-NEXT: s_waitcnt lgkmcnt(8)
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v16, 16, v14
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v16, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v14, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v11
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[18:19], v[20:21] offset0:16 offset1:17
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v0, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v20, v11, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v14, 16, v13
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:14 offset1:15
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v14, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v13, v13, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[13:14], v[16:17] offset0:12 offset1:13
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v13, 16, v12
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v21, 31, v20
; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v13, v13, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v12, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v12, v0, 0, 16
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[13:14] offset0:10 offset1:11
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v0, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v11, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[20:21], v[18:19] offset0:14 offset1:15
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v0, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v10, v10, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v7
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v20, 16, v8
-; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v9
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[18:19], v[16:17] offset0:8 offset1:9
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v16, v9, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v9, v10, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v5, v5, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v1, v20, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v3, v3, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v20, v7, 0, 16
-; GFX9-NO-DS128-NEXT: v_bfe_i32 v7, v8, 0, 16
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v9
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[10:11], v[18:19] offset0:12 offset1:13
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v10, v0, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v9, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v3, 16, v7
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v11, 31, v10
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v8
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[18:19], v[10:11] offset0:10 offset1:11
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v9, v3, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v18, v0, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v3, v8, 0, 16
+; GFX9-NO-DS128-NEXT: v_lshrrev_b32_e32 v1, 16, v4
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v20, v4, 0, 16
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v1, v1, 0, 16
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[3:4], v[18:19] offset0:8 offset1:9
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v3, v5, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v5, v6, 0, 16
+; GFX9-NO-DS128-NEXT: v_bfe_i32 v7, v7, 0, 16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v2, 31, v1
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v21, 31, v20
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v4, 31, v3
+; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v6, 31, v5
; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v8, 31, v7
-; GFX9-NO-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[9:10], v[12:13] offset0:6 offset1:7
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[16:17], v[3:4] offset0:4 offset1:5
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[7:8], v[1:2] offset0:2 offset1:3
-; GFX9-NO-DS128-NEXT: ds_write2_b64 v15, v[20:21], v[5:6] offset1:1
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[7:8], v[9:10] offset0:6 offset1:7
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[5:6], v[16:17] offset0:4 offset1:5
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[3:4], v[14:15] offset0:2 offset1:3
+; GFX9-NO-DS128-NEXT: ds_write2_b64 v12, v[20:21], v[1:2] offset1:1
; GFX9-NO-DS128-NEXT: s_endpgm
;
; EG-LABEL: local_sextload_v32i16_to_v32i64:
@@ -8684,27 +8744,27 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; VI-DS128-NEXT: s_waitcnt lgkmcnt(0)
; VI-DS128-NEXT: v_mov_b32_e32 v4, s1
; VI-DS128-NEXT: ds_read_b128 v[0:3], v4 offset:48
-; VI-DS128-NEXT: ds_read_b128 v[9:12], v4 offset:32
; VI-DS128-NEXT: v_mov_b32_e32 v8, s0
-; VI-DS128-NEXT: ds_read_b128 v[17:20], v4 offset:16
+; VI-DS128-NEXT: ds_read_b128 v[9:12], v4 offset:32
+; VI-DS128-NEXT: ds_read_b128 v[13:16], v4 offset:16
; VI-DS128-NEXT: ds_read_b128 v[4:7], v4
; VI-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; VI-DS128-NEXT: v_bfe_i32 v13, v2, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v17, v2, 0, 16
; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; VI-DS128-NEXT: v_bfe_i32 v15, v2, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
+; VI-DS128-NEXT: v_bfe_i32 v19, v2, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:224
-; VI-DS128-NEXT: v_bfe_i32 v15, v2, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v13, v3, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
+; VI-DS128-NEXT: ds_write_b128 v8, v[17:20] offset:224
+; VI-DS128-NEXT: v_bfe_i32 v19, v2, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v17, v3, 0, 16
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v0
-; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:240
-; VI-DS128-NEXT: v_bfe_i32 v15, v2, 0, 16
+; VI-DS128-NEXT: ds_write_b128 v8, v[17:20] offset:240
+; VI-DS128-NEXT: v_bfe_i32 v19, v2, 0, 16
; VI-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v1
-; VI-DS128-NEXT: v_bfe_i32 v13, v0, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v17, v0, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v0, v1, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
@@ -8737,33 +8797,30 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; VI-DS128-NEXT: v_bfe_i32 v11, v5, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v19
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v15
; VI-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:144
-; VI-DS128-NEXT: v_bfe_i32 v9, v19, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v9, v15, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v11, v5, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v20
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v16
; VI-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:96
-; VI-DS128-NEXT: v_bfe_i32 v9, v20, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v9, v16, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v11, v5, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v17
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v13
; VI-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:112
-; VI-DS128-NEXT: v_bfe_i32 v9, v17, 0, 16
+; VI-DS128-NEXT: v_bfe_i32 v9, v13, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v11, v5, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
; VI-DS128-NEXT: ds_write_b128 v8, v[9:12] offset:64
; VI-DS128-NEXT: v_bfe_i32 v9, v4, 0, 16
; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v4
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
; VI-DS128-NEXT: v_bfe_i32 v11, v4, 0, 16
-; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v18
-; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:192
-; VI-DS128-NEXT: v_bfe_i32 v13, v18, 0, 16
+; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v14
+; VI-DS128-NEXT: v_bfe_i32 v13, v14, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v15, v4, 0, 16
; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
@@ -8771,18 +8828,21 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; VI-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
; VI-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v7
; VI-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v15, v4, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
; VI-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
-; VI-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
; VI-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; VI-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; VI-DS128-NEXT: v_ashrrev_i32_e32 v10, 31, v9
; VI-DS128-NEXT: v_ashrrev_i32_e32 v12, 31, v11
; VI-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
; VI-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; VI-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; VI-DS128-NEXT: ds_write_b128 v8, v[17:20] offset:192
; VI-DS128-NEXT: ds_write_b128 v8, v[4:7] offset:32
; VI-DS128-NEXT: ds_write_b128 v8, v[13:16] offset:48
; VI-DS128-NEXT: ds_write_b128 v8, v[9:12]
@@ -8793,112 +8853,110 @@ define amdgpu_kernel void @local_sextload_v32i16_to_v32i64(ptr addrspace(3) %out
; GFX9-DS128: ; %bb.0:
; GFX9-DS128-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(0)
-; GFX9-DS128-NEXT: v_mov_b32_e32 v13, s1
-; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v13 offset:48
-; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v13 offset:32
-; GFX9-DS128-NEXT: v_mov_b32_e32 v12, s0
-; GFX9-DS128-NEXT: ds_read_b128 v[8:11], v13
-; GFX9-DS128-NEXT: ds_read_b128 v[18:21], v13 offset:16
+; GFX9-DS128-NEXT: v_mov_b32_e32 v9, s1
+; GFX9-DS128-NEXT: ds_read_b128 v[0:3], v9 offset:48
+; GFX9-DS128-NEXT: v_mov_b32_e32 v8, s0
+; GFX9-DS128-NEXT: ds_read_b128 v[4:7], v9
+; GFX9-DS128-NEXT: ds_read_b128 v[10:13], v9 offset:16
+; GFX9-DS128-NEXT: ds_read_b128 v[18:21], v9 offset:32
; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(3)
-; GFX9-DS128-NEXT: v_bfe_i32 v14, v6, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v6
-; GFX9-DS128-NEXT: v_bfe_i32 v16, v6, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v2, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v2
+; GFX9-DS128-NEXT: v_bfe_i32 v16, v2, 0, 16
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v7
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[14:17] offset:224
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v6, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v7, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v4
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:240
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v6, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v6, 16, v5
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v4, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v5, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v6, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[4:7] offset:208
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(5)
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v2, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v2
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v2, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v3
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[4:7] offset:160
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v3, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v2, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GFX9-DS128-NEXT: v_bfe_i32 v2, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[4:7] offset:176
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v0, 0, 16
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:224
+; GFX9-DS128-NEXT: v_bfe_i32 v16, v2, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v3, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v0
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:240
+; GFX9-DS128-NEXT: v_bfe_i32 v16, v2, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v1
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v0, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v1, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(6)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v9
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[2:5] offset:128
-; GFX9-DS128-NEXT: v_bfe_i32 v4, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v1
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:192
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v1, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(7)
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v20
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:144
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v20, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v21
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:96
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v21, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v18
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:112
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v18, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v8
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_bfe_i32 v6, v8, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v8, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v19
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:64
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v19, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v11
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:80
-; GFX9-DS128-NEXT: v_bfe_i32 v15, v0, 0, 16
-; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v0, 16, v10
-; GFX9-DS128-NEXT: v_bfe_i32 v17, v10, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v19, v0, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v2, v9, 0, 16
-; GFX9-DS128-NEXT: v_bfe_i32 v13, v11, 0, 16
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v18, 31, v17
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v20, 31, v19
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:208
+; GFX9-DS128-NEXT: s_waitcnt lgkmcnt(3)
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v20
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v20, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:160
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v21
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v21, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:176
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v18
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v18, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:128
+; GFX9-DS128-NEXT: v_bfe_i32 v0, v5, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v2, 16, v5
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v19
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:192
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v19, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v16, v5, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v12
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:144
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v12, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v16, v5, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v13
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:96
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v13, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v16, v5, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v5, 16, v10
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[14:17] offset:112
+; GFX9-DS128-NEXT: v_bfe_i32 v12, v10, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v5, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[12:15] offset:64
+; GFX9-DS128-NEXT: v_bfe_i32 v12, v4, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v4
+; GFX9-DS128-NEXT: v_bfe_i32 v14, v4, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v11
+; GFX9-DS128-NEXT: v_bfe_i32 v16, v11, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v18, v4, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[16:19] offset:80
+; GFX9-DS128-NEXT: v_bfe_i32 v16, v7, 0, 16
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v4, 16, v7
+; GFX9-DS128-NEXT: v_lshrrev_b32_e32 v7, 16, v6
+; GFX9-DS128-NEXT: v_bfe_i32 v2, v2, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v18, v4, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v4, v6, 0, 16
+; GFX9-DS128-NEXT: v_bfe_i32 v6, v7, 0, 16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v3, 31, v2
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v15, 31, v14
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v17, 31, v16
+; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v19, 31, v18
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v5, 31, v4
; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v9, 31, v8
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v14, 31, v13
-; GFX9-DS128-NEXT: v_ashrrev_i32_e32 v16, 31, v15
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[17:20] offset:32
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[13:16] offset:48
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[6:9]
-; GFX9-DS128-NEXT: ds_write_b128 v12, v[2:5] offset:16
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[4:7] offset:32
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[16:19] offset:48
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[12:15]
+; GFX9-DS128-NEXT: ds_write_b128 v8, v[0:3] offset:16
; GFX9-DS128-NEXT: s_endpgm
%load = load <32 x i16>, ptr addrspace(3) %in
%ext = sext <32 x i16> %load to <32 x i64>
diff --git a/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll b/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
index c9615f478e5b5..66be3ef174055 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local-redundant-copies.ll
@@ -30,22 +30,22 @@ define amdgpu_vs void @test(ptr addrspace(8) inreg %arg1, ptr addrspace(3) %arg2
define amdgpu_vs void @test_2(ptr addrspace(8) inreg %arg1, i32 %arg2, i32 inreg %arg3, ptr addrspace(3) %arg4) {
; CHECK-LABEL: test_2:
; CHECK: ; %bb.0:
-; CHECK-NEXT: v_add_i32_e32 v3, vcc, 20, v1
-; CHECK-NEXT: v_add_i32_e32 v2, vcc, 16, v1
-; CHECK-NEXT: v_add_i32_e32 v4, vcc, 28, v1
-; CHECK-NEXT: v_add_i32_e32 v6, vcc, 24, v1
-; CHECK-NEXT: v_add_i32_e32 v7, vcc, 12, v1
-; CHECK-NEXT: v_add_i32_e32 v8, vcc, 8, v1
-; CHECK-NEXT: v_add_i32_e32 v10, vcc, 4, v1
+; CHECK-NEXT: v_add_i32_e32 v2, vcc, 20, v1
+; CHECK-NEXT: v_add_i32_e32 v4, vcc, 16, v1
+; CHECK-NEXT: v_add_i32_e32 v5, vcc, 28, v1
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, 24, v1
+; CHECK-NEXT: v_add_i32_e32 v6, vcc, 12, v1
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, 8, v1
+; CHECK-NEXT: v_add_i32_e32 v11, vcc, 4, v1
; CHECK-NEXT: s_mov_b32 m0, -1
-; CHECK-NEXT: ds_read_b32 v2, v2
-; CHECK-NEXT: ds_read_b32 v5, v4
-; CHECK-NEXT: ds_read_b32 v4, v6
-; CHECK-NEXT: ds_read_b32 v9, v7
-; CHECK-NEXT: ds_read_b32 v8, v8
-; CHECK-NEXT: ds_read_b32 v7, v10
+; CHECK-NEXT: ds_read_b32 v3, v2
+; CHECK-NEXT: ds_read_b32 v2, v4
+; CHECK-NEXT: ds_read_b32 v5, v5
+; CHECK-NEXT: ds_read_b32 v9, v6
+; CHECK-NEXT: ds_read_b32 v8, v7
+; CHECK-NEXT: ds_read_b32 v7, v11
; CHECK-NEXT: ds_read_b32 v6, v1
-; CHECK-NEXT: ds_read_b32 v3, v3
+; CHECK-NEXT: ds_read_b32 v4, v10
; CHECK-NEXT: s_waitcnt lgkmcnt(1)
; CHECK-NEXT: tbuffer_store_format_xyzw v[6:9], v0, s[0:3], s4 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen glc slc
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
@@ -70,35 +70,36 @@ define amdgpu_vs void @test_3(i32 inreg %arg1, i32 inreg %arg2, ptr addrspace(8)
; CHECK-NEXT: v_add_i32_e32 v3, vcc, 8, v1
; CHECK-NEXT: v_add_i32_e32 v4, vcc, 4, v1
; CHECK-NEXT: v_add_i32_e32 v7, vcc, 20, v1
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, 16, v1
-; CHECK-NEXT: v_mov_b32_e32 v10, s0
-; CHECK-NEXT: v_add_i32_e32 v11, vcc, 12, v2
-; CHECK-NEXT: v_add_i32_e32 v12, vcc, 8, v2
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, 16, v1
+; CHECK-NEXT: v_mov_b32_e32 v9, s0
+; CHECK-NEXT: v_add_i32_e32 v10, vcc, 12, v2
+; CHECK-NEXT: v_add_i32_e32 v11, vcc, 8, v2
+; CHECK-NEXT: v_add_i32_e32 v12, vcc, 4, v2
; CHECK-NEXT: s_mov_b32 m0, -1
; CHECK-NEXT: ds_read_b32 v6, v0
; CHECK-NEXT: ds_read_b32 v5, v3
; CHECK-NEXT: ds_read_b32 v4, v4
-; CHECK-NEXT: ds_read_b32 v8, v7
-; CHECK-NEXT: ds_read_b32 v7, v9
; CHECK-NEXT: ds_read_b32 v3, v1
-; CHECK-NEXT: v_add_i32_e32 v0, vcc, 4, v2
-; CHECK-NEXT: v_add_i32_e32 v1, vcc, 20, v2
-; CHECK-NEXT: v_add_i32_e32 v9, vcc, 16, v2
+; CHECK-NEXT: ds_read_b32 v1, v7
+; CHECK-NEXT: ds_read_b32 v0, v8
+; CHECK-NEXT: v_add_i32_e32 v7, vcc, 20, v2
+; CHECK-NEXT: v_add_i32_e32 v8, vcc, 16, v2
+; CHECK-NEXT: s_waitcnt lgkmcnt(2)
+; CHECK-NEXT: tbuffer_store_format_xyzw v[3:6], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:264 glc slc
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: tbuffer_store_format_xyzw v[3:6], v10, s[4:7], s1 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:264 glc slc
-; CHECK-NEXT: tbuffer_store_format_xy v[7:8], v10, s[4:7], s1 format:[BUF_DATA_FORMAT_INVALID,BUF_NUM_FORMAT_UINT] idxen offset:280 glc slc
+; CHECK-NEXT: tbuffer_store_format_xy v[0:1], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_INVALID,BUF_NUM_FORMAT_UINT] idxen offset:280 glc slc
; CHECK-NEXT: s_waitcnt expcnt(1)
-; CHECK-NEXT: ds_read_b32 v5, v11
-; CHECK-NEXT: ds_read_b32 v4, v12
-; CHECK-NEXT: ds_read_b32 v3, v0
-; CHECK-NEXT: ds_read_b32 v1, v1
-; CHECK-NEXT: ds_read_b32 v0, v9
+; CHECK-NEXT: ds_read_b32 v5, v10
+; CHECK-NEXT: ds_read_b32 v4, v11
+; CHECK-NEXT: ds_read_b32 v3, v12
; CHECK-NEXT: ds_read_b32 v2, v2
-; CHECK-NEXT: s_waitcnt lgkmcnt(1)
-; CHECK-NEXT: exp mrt0 off, off, off, off
+; CHECK-NEXT: s_waitcnt expcnt(0)
+; CHECK-NEXT: ds_read_b32 v1, v7
+; CHECK-NEXT: ds_read_b32 v0, v8
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: tbuffer_store_format_xyzw v[2:5], v10, s[4:7], s1 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:240 glc slc
-; CHECK-NEXT: tbuffer_store_format_xy v[0:1], v10, s[4:7], s1 format:[BUF_DATA_FORMAT_INVALID,BUF_NUM_FORMAT_UINT] idxen offset:256 glc slc
+; CHECK-NEXT: exp mrt0 off, off, off, off
+; CHECK-NEXT: tbuffer_store_format_xyzw v[2:5], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_32_32_32,BUF_NUM_FORMAT_UINT] idxen offset:240 glc slc
+; CHECK-NEXT: tbuffer_store_format_xy v[0:1], v9, s[4:7], s1 format:[BUF_DATA_FORMAT_INVALID,BUF_NUM_FORMAT_UINT] idxen offset:256 glc slc
; CHECK-NEXT: s_endpgm
%load1 = load <6 x float>, ptr addrspace(3) %arg5, align 4
%vec11 = shufflevector <6 x float> %load1, <6 x float> poison, <4 x i32> <i32 0, i32 1, i32 2, i32 3>
diff --git a/llvm/test/CodeGen/AMDGPU/load-local.128.ll b/llvm/test/CodeGen/AMDGPU/load-local.128.ll
index 5b6af7654f7e9..36bacd80b8a5e 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local.128.ll
@@ -96,52 +96,59 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_u8 v1, v0 offset:1
-; GFX7-NEXT: ds_read_u8 v2, v0 offset:6
-; GFX7-NEXT: ds_read_u8 v3, v0 offset:4
+; GFX7-NEXT: ds_read_u8 v2, v0
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:3
; GFX7-NEXT: ds_read_u8 v4, v0 offset:2
-; GFX7-NEXT: ds_read_u8 v5, v0
-; GFX7-NEXT: ds_read_u8 v6, v0 offset:3
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:5
-; GFX7-NEXT: ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT: s_waitcnt lgkmcnt(7)
+; GFX7-NEXT: ds_read_u8 v6, v0 offset:7
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(3)
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v6
-; GFX7-NEXT: v_or_b32_e32 v4, v5, v4
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX7-NEXT: v_or_b32_e32 v4, v4, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v7
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
-; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v8
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:9
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v3
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:5
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_or_b32_e32 v4, v2, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v6
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:9
; GFX7-NEXT: ds_read_u8 v6, v0 offset:11
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:13
-; GFX7-NEXT: ds_read_u8 v8, v0 offset:15
-; GFX7-NEXT: ds_read_u8 v9, v0 offset:14
-; GFX7-NEXT: ds_read_u8 v10, v0 offset:12
-; GFX7-NEXT: ds_read_u8 v11, v0 offset:10
-; GFX7-NEXT: ds_read_u8 v0, v0 offset:8
-; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v7
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:8
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:10
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX7-NEXT: s_waitcnt lgkmcnt(7)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v5
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v3
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v6
+; GFX7-NEXT: s_waitcnt lgkmcnt(1)
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v6
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v11
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v8
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 8, v7
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v9
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v10
+; GFX7-NEXT: v_or_b32_e32 v3, v3, v7
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:13
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:15
+; GFX7-NEXT: ds_read_u8 v6, v0 offset:12
+; GFX7-NEXT: ds_read_u8 v0, v0 offset:14
; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v0
+; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 8, v5
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v7
+; GFX7-NEXT: s_waitcnt lgkmcnt(0)
+; GFX7-NEXT: v_or_b32_e32 v0, v5, v0
+; GFX7-NEXT: v_or_b32_e32 v3, v3, v6
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v3, v0, v3
; GFX7-NEXT: v_mov_b32_e32 v0, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -149,73 +156,76 @@ define <4 x i32> @load_lds_v4i32_align1(ptr addrspace(3) %ptr) {
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 5, v0
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, 4, v0
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, 7, v0
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, 6, v0
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 9, v0
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, 8, v0
-; GFX6-NEXT: v_add_i32_e32 v7, vcc, 11, v0
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: ds_read_u8 v1, v1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 4, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 7, v0
; GFX6-NEXT: ds_read_u8 v2, v2
; GFX6-NEXT: ds_read_u8 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 6, v0
; GFX6-NEXT: ds_read_u8 v4, v4
-; GFX6-NEXT: ds_read_u8 v5, v5
-; GFX6-NEXT: ds_read_u8 v6, v6
-; GFX6-NEXT: ds_read_u8 v7, v7
-; GFX6-NEXT: ds_read_u8 v8, v0
-; GFX6-NEXT: s_waitcnt lgkmcnt(7)
+; GFX6-NEXT: s_waitcnt lgkmcnt(3)
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX6-NEXT: s_waitcnt lgkmcnt(6)
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: s_waitcnt lgkmcnt(5)
+; GFX6-NEXT: s_waitcnt lgkmcnt(1)
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v3
-; GFX6-NEXT: s_waitcnt lgkmcnt(4)
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 9, v0
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 11, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT: ds_read_u8 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 8, v0
+; GFX6-NEXT: ds_read_u8 v5, v5
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 10, v0
+; GFX6-NEXT: ds_read_u8 v4, v4
+; GFX6-NEXT: ds_read_u8 v6, v6
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
; GFX6-NEXT: s_waitcnt lgkmcnt(3)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(2)
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 8, v5
; GFX6-NEXT: s_waitcnt lgkmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 8, v7
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, 10, v0
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 13, v0
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, 12, v0
-; GFX6-NEXT: v_add_i32_e32 v7, vcc, 15, v0
-; GFX6-NEXT: v_add_i32_e32 v9, vcc, 14, v0
-; GFX6-NEXT: v_add_i32_e32 v10, vcc, 3, v0
-; GFX6-NEXT: v_add_i32_e32 v11, vcc, 2, v0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v0
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_or_b32_e32 v3, v3, v6
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 13, v0
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 15, v0
; GFX6-NEXT: ds_read_u8 v4, v4
-; GFX6-NEXT: ds_read_u8 v5, v5
; GFX6-NEXT: ds_read_u8 v6, v6
+; GFX6-NEXT: v_add_i32_e32 v7, vcc, 14, v0
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 12, v0
; GFX6-NEXT: ds_read_u8 v7, v7
-; GFX6-NEXT: ds_read_u8 v9, v9
-; GFX6-NEXT: ds_read_u8 v10, v10
-; GFX6-NEXT: ds_read_u8 v11, v11
-; GFX6-NEXT: ds_read_u8 v0, v0
-; GFX6-NEXT: s_waitcnt lgkmcnt(7)
-; GFX6-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT: ds_read_u8 v5, v5
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: s_waitcnt lgkmcnt(4)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 8, v7
; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 8, v5
; GFX6-NEXT: s_waitcnt lgkmcnt(3)
-; GFX6-NEXT: v_or_b32_e32 v4, v4, v9
-; GFX6-NEXT: v_or_b32_e32 v3, v3, v6
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 8, v4
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 8, v6
+; GFX6-NEXT: s_waitcnt lgkmcnt(1)
+; GFX6-NEXT: v_or_b32_e32 v4, v4, v7
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_or_b32_e32 v3, v3, v5
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
; GFX6-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 3, v0
+; GFX6-NEXT: ds_read_u8 v4, v4
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 2, v0
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 1, v0
+; GFX6-NEXT: ds_read_u8 v5, v5
+; GFX6-NEXT: ds_read_u8 v6, v6
+; GFX6-NEXT: ds_read_u8 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(3)
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 8, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(2)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 8, v10
+; GFX6-NEXT: v_or_b32_e32 v4, v4, v5
; GFX6-NEXT: s_waitcnt lgkmcnt(1)
-; GFX6-NEXT: v_or_b32_e32 v4, v4, v11
-; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX6-NEXT: v_lshlrev_b32_e32 v5, 8, v6
; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_or_b32_e32 v0, v5, v0
; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
@@ -332,60 +342,63 @@ define <4 x i32> @load_lds_v4i32_align2(ptr addrspace(3) %ptr) {
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_u16 v1, v0 offset:2
-; GFX7-NEXT: ds_read_u16 v3, v0 offset:12
-; GFX7-NEXT: ds_read_u16 v2, v0 offset:8
-; GFX7-NEXT: ds_read_u16 v4, v0 offset:4
-; GFX7-NEXT: ds_read_u16 v5, v0
-; GFX7-NEXT: ds_read_u16 v6, v0 offset:6
-; GFX7-NEXT: ds_read_u16 v7, v0 offset:10
-; GFX7-NEXT: ds_read_u16 v8, v0 offset:14
-; GFX7-NEXT: s_waitcnt lgkmcnt(7)
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX7-NEXT: ds_read_u16 v3, v0
+; GFX7-NEXT: ds_read_u16 v2, v0 offset:6
+; GFX7-NEXT: ds_read_u16 v6, v0 offset:14
+; GFX7-NEXT: ds_read_u16 v5, v0 offset:8
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(3)
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v5
+; GFX7-NEXT: v_or_b32_e32 v4, v1, v3
+; GFX7-NEXT: ds_read_u16 v1, v0 offset:4
+; GFX7-NEXT: ds_read_u16 v3, v0 offset:10
+; GFX7-NEXT: ds_read_u16 v0, v0 offset:12
+; GFX7-NEXT: s_waitcnt lgkmcnt(5)
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v6
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v7
-; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 16, v8
-; GFX7-NEXT: v_or_b32_e32 v3, v4, v3
+; GFX7-NEXT: v_or_b32_e32 v3, v3, v0
+; GFX7-NEXT: v_mov_b32_e32 v0, v4
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: load_lds_v4i32_align2:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 6, v0
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, 4, v0
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, 10, v0
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, 8, v0
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 14, v0
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, 12, v0
-; GFX6-NEXT: v_add_i32_e32 v7, vcc, 2, v0
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: ds_read_u16 v1, v1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 4, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 10, v0
; GFX6-NEXT: ds_read_u16 v2, v2
; GFX6-NEXT: ds_read_u16 v3, v3
-; GFX6-NEXT: ds_read_u16 v4, v4
-; GFX6-NEXT: ds_read_u16 v5, v5
-; GFX6-NEXT: ds_read_u16 v6, v6
-; GFX6-NEXT: ds_read_u16 v7, v7
-; GFX6-NEXT: ds_read_u16 v0, v0
-; GFX6-NEXT: s_waitcnt lgkmcnt(7)
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX6-NEXT: s_waitcnt lgkmcnt(6)
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 14, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(1)
; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: s_waitcnt lgkmcnt(5)
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 8, v0
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 2, v0
+; GFX6-NEXT: ds_read_u16 v3, v3
+; GFX6-NEXT: ds_read_u16 v4, v4
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 12, v0
+; GFX6-NEXT: ds_read_u16 v6, v6
+; GFX6-NEXT: ds_read_u16 v5, v5
+; GFX6-NEXT: ds_read_u16 v0, v0
; GFX6-NEXT: s_waitcnt lgkmcnt(4)
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(3)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v4
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v6
; GFX6-NEXT: s_waitcnt lgkmcnt(1)
-; GFX6-NEXT: v_lshlrev_b32_e32 v4, 16, v7
-; GFX6-NEXT: v_or_b32_e32 v3, v3, v6
+; GFX6-NEXT: v_or_b32_e32 v3, v3, v5
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
@@ -458,14 +471,16 @@ define <4 x i32> @load_lds_v4i32_align4(ptr addrspace(3) %ptr) {
; GFX6-LABEL: load_lds_v4i32_align4:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 4, v0
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v0
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, 12, v0
-; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v2
-; GFX6-NEXT: ds_read_b32 v3, v3
-; GFX6-NEXT: ds_read_b32 v0, v0
+; GFX6-NEXT: ds_read_b32 v4, v0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 12, v0
; GFX6-NEXT: ds_read_b32 v1, v1
+; GFX6-NEXT: ds_read_b32 v2, v2
+; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(3)
+; GFX6-NEXT: v_mov_b32_e32 v0, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/load-local.96.ll b/llvm/test/CodeGen/AMDGPU/load-local.96.ll
index 509aba49893f6..ae84e7c4790f2 100644
--- a/llvm/test/CodeGen/AMDGPU/load-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-local.96.ll
@@ -87,42 +87,45 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_u8 v1, v0 offset:1
-; GFX7-NEXT: ds_read_u8 v2, v0 offset:6
-; GFX7-NEXT: ds_read_u8 v4, v0 offset:4
-; GFX7-NEXT: ds_read_u8 v3, v0 offset:2
-; GFX7-NEXT: ds_read_u8 v5, v0
-; GFX7-NEXT: ds_read_u8 v6, v0 offset:3
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:5
-; GFX7-NEXT: ds_read_u8 v8, v0 offset:7
-; GFX7-NEXT: s_waitcnt lgkmcnt(7)
+; GFX7-NEXT: ds_read_u8 v2, v0
+; GFX7-NEXT: ds_read_u8 v3, v0 offset:3
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:2
+; GFX7-NEXT: ds_read_u8 v6, v0 offset:7
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(3)
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v2
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_lshlrev_b32_e32 v5, 8, v6
-; GFX7-NEXT: v_or_b32_e32 v3, v5, v3
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX7-NEXT: v_or_b32_e32 v3, v3, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v7
-; GFX7-NEXT: ds_read_u8 v5, v0 offset:9
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:5
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:4
+; GFX7-NEXT: ds_read_u8 v7, v0 offset:6
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX7-NEXT: v_or_b32_e32 v3, v2, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 8, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v6
+; GFX7-NEXT: ds_read_u8 v4, v0 offset:9
; GFX7-NEXT: ds_read_u8 v6, v0 offset:11
-; GFX7-NEXT: ds_read_u8 v7, v0 offset:10
-; GFX7-NEXT: ds_read_u8 v0, v0 offset:8
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v4
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v5
+; GFX7-NEXT: ds_read_u8 v5, v0 offset:8
+; GFX7-NEXT: ds_read_u8 v0, v0 offset:10
; GFX7-NEXT: s_waitcnt lgkmcnt(4)
-; GFX7-NEXT: v_lshlrev_b32_e32 v4, 8, v8
-; GFX7-NEXT: v_or_b32_e32 v2, v4, v2
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v7
; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX7-NEXT: v_or_b32_e32 v1, v2, v1
; GFX7-NEXT: s_waitcnt lgkmcnt(3)
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v4
+; GFX7-NEXT: s_waitcnt lgkmcnt(2)
+; GFX7-NEXT: v_lshlrev_b32_e32 v4, 8, v6
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_or_b32_e32 v0, v2, v0
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 8, v6
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v7
-; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v2
-; GFX7-NEXT: v_or_b32_e32 v2, v2, v0
+; GFX7-NEXT: v_or_b32_e32 v0, v4, v0
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v5
+; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX7-NEXT: v_or_b32_e32 v2, v0, v2
; GFX7-NEXT: v_mov_b32_e32 v0, v3
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
@@ -130,57 +133,58 @@ define <3 x i32> @load_lds_v3i32_align1(ptr addrspace(3) %ptr) {
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 5, v0
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, 4, v0
-; GFX6-NEXT: v_add_i32_e32 v3, vcc, 7, v0
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, 6, v0
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 9, v0
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, 8, v0
-; GFX6-NEXT: v_add_i32_e32 v7, vcc, 11, v0
; GFX6-NEXT: s_mov_b32 m0, -1
; GFX6-NEXT: ds_read_u8 v1, v1
+; GFX6-NEXT: v_add_i32_e32 v2, vcc, 4, v0
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 7, v0
; GFX6-NEXT: ds_read_u8 v2, v2
; GFX6-NEXT: ds_read_u8 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 6, v0
; GFX6-NEXT: ds_read_u8 v4, v4
-; GFX6-NEXT: ds_read_u8 v5, v5
-; GFX6-NEXT: ds_read_u8 v6, v6
-; GFX6-NEXT: ds_read_u8 v7, v7
-; GFX6-NEXT: ds_read_u8 v8, v0
-; GFX6-NEXT: s_waitcnt lgkmcnt(7)
+; GFX6-NEXT: s_waitcnt lgkmcnt(3)
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 8, v1
-; GFX6-NEXT: s_waitcnt lgkmcnt(6)
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
-; GFX6-NEXT: s_waitcnt lgkmcnt(5)
+; GFX6-NEXT: s_waitcnt lgkmcnt(1)
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v3
-; GFX6-NEXT: s_waitcnt lgkmcnt(4)
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 9, v0
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 11, v0
+; GFX6-NEXT: ds_read_u8 v3, v3
+; GFX6-NEXT: ds_read_u8 v5, v5
+; GFX6-NEXT: v_add_i32_e32 v6, vcc, 10, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, 10, v0
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 8, v0
+; GFX6-NEXT: ds_read_u8 v6, v6
; GFX6-NEXT: ds_read_u8 v4, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX6-NEXT: v_or_b32_e32 v1, v2, v1
-; GFX6-NEXT: s_waitcnt lgkmcnt(4)
-; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v5
; GFX6-NEXT: s_waitcnt lgkmcnt(3)
-; GFX6-NEXT: v_or_b32_e32 v2, v2, v6
-; GFX6-NEXT: v_add_i32_e32 v5, vcc, 3, v0
-; GFX6-NEXT: v_add_i32_e32 v6, vcc, 2, v0
-; GFX6-NEXT: v_add_i32_e32 v0, vcc, 1, v0
-; GFX6-NEXT: ds_read_u8 v5, v5
-; GFX6-NEXT: ds_read_u8 v6, v6
-; GFX6-NEXT: ds_read_u8 v0, v0
-; GFX6-NEXT: s_waitcnt lgkmcnt(5)
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 8, v7
-; GFX6-NEXT: s_waitcnt lgkmcnt(3)
-; GFX6-NEXT: v_or_b32_e32 v3, v3, v4
-; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: v_lshlrev_b32_e32 v2, 8, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(2)
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 8, v5
; GFX6-NEXT: s_waitcnt lgkmcnt(1)
; GFX6-NEXT: v_or_b32_e32 v3, v3, v6
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
-; GFX6-NEXT: v_lshlrev_b32_e32 v0, 8, v0
+; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
+; GFX6-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX6-NEXT: v_add_i32_e32 v3, vcc, 3, v0
+; GFX6-NEXT: ds_read_u8 v3, v3
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 2, v0
+; GFX6-NEXT: v_add_i32_e32 v5, vcc, 1, v0
+; GFX6-NEXT: ds_read_u8 v4, v4
+; GFX6-NEXT: ds_read_u8 v5, v5
+; GFX6-NEXT: ds_read_u8 v0, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(3)
+; GFX6-NEXT: v_lshlrev_b32_e32 v3, 8, v3
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-NEXT: v_or_b32_e32 v3, v3, v4
+; GFX6-NEXT: s_waitcnt lgkmcnt(1)
+; GFX6-NEXT: v_lshlrev_b32_e32 v4, 8, v5
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v3
-; GFX6-NEXT: v_or_b32_e32 v0, v0, v8
+; GFX6-NEXT: s_waitcnt lgkmcnt(0)
+; GFX6-NEXT: v_or_b32_e32 v0, v4, v0
; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
@@ -275,37 +279,39 @@ define <3 x i32> @load_lds_v3i32_align2(ptr addrspace(3) %ptr) {
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: s_mov_b32 m0, -1
; GFX7-NEXT: ds_read_u16 v1, v0 offset:2
-; GFX7-NEXT: ds_read_u16 v2, v0 offset:8
-; GFX7-NEXT: ds_read_u16 v3, v0 offset:4
-; GFX7-NEXT: ds_read_u16 v4, v0
-; GFX7-NEXT: ds_read_u16 v5, v0 offset:6
-; GFX7-NEXT: ds_read_u16 v6, v0 offset:10
+; GFX7-NEXT: ds_read_u16 v2, v0
+; GFX7-NEXT: ds_read_u16 v3, v0 offset:6
+; GFX7-NEXT: ds_read_u16 v5, v0 offset:10
+; GFX7-NEXT: ds_read_u16 v4, v0 offset:4
+; GFX7-NEXT: ds_read_u16 v6, v0 offset:8
; GFX7-NEXT: s_waitcnt lgkmcnt(5)
-; GFX7-NEXT: v_lshlrev_b32_e32 v0, 16, v1
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX7-NEXT: s_waitcnt lgkmcnt(4)
+; GFX7-NEXT: v_or_b32_e32 v0, v1, v2
+; GFX7-NEXT: s_waitcnt lgkmcnt(3)
+; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v3
; GFX7-NEXT: s_waitcnt lgkmcnt(2)
-; GFX7-NEXT: v_or_b32_e32 v0, v0, v4
+; GFX7-NEXT: v_lshlrev_b32_e32 v2, 16, v5
; GFX7-NEXT: s_waitcnt lgkmcnt(1)
-; GFX7-NEXT: v_lshlrev_b32_e32 v1, 16, v5
-; GFX7-NEXT: v_or_b32_e32 v1, v1, v3
+; GFX7-NEXT: v_or_b32_e32 v1, v1, v4
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
-; GFX7-NEXT: v_lshlrev_b32_e32 v3, 16, v6
-; GFX7-NEXT: v_or_b32_e32 v2, v3, v2
+; GFX7-NEXT: v_or_b32_e32 v2, v2, v6
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX6-LABEL: load_lds_v3i32_align2:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX6-NEXT: v_add_i32_e32 v1, vcc, 6, v0
+; GFX6-NEXT: s_mov_b32 m0, -1
+; GFX6-NEXT: ds_read_u16 v1, v1
; GFX6-NEXT: v_add_i32_e32 v2, vcc, 4, v0
; GFX6-NEXT: v_add_i32_e32 v3, vcc, 10, v0
-; GFX6-NEXT: v_add_i32_e32 v4, vcc, 8, v0
; GFX6-NEXT: v_add_i32_e32 v5, vcc, 2, v0
-; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_u16 v1, v1
; GFX6-NEXT: ds_read_u16 v2, v2
; GFX6-NEXT: ds_read_u16 v3, v3
-; GFX6-NEXT: ds_read_u16 v4, v4
+; GFX6-NEXT: v_add_i32_e32 v4, vcc, 8, v0
; GFX6-NEXT: ds_read_u16 v5, v5
+; GFX6-NEXT: ds_read_u16 v4, v4
; GFX6-NEXT: ds_read_u16 v0, v0
; GFX6-NEXT: s_waitcnt lgkmcnt(5)
; GFX6-NEXT: v_lshlrev_b32_e32 v1, 16, v1
@@ -313,8 +319,9 @@ define <3 x i32> @load_lds_v3i32_align2(ptr addrspace(3) %ptr) {
; GFX6-NEXT: v_or_b32_e32 v1, v1, v2
; GFX6-NEXT: s_waitcnt lgkmcnt(3)
; GFX6-NEXT: v_lshlrev_b32_e32 v2, 16, v3
-; GFX6-NEXT: s_waitcnt lgkmcnt(1)
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
; GFX6-NEXT: v_lshlrev_b32_e32 v3, 16, v5
+; GFX6-NEXT: s_waitcnt lgkmcnt(1)
; GFX6-NEXT: v_or_b32_e32 v2, v2, v4
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_or_b32_e32 v0, v3, v0
@@ -380,12 +387,14 @@ define <3 x i32> @load_lds_v3i32_align4(ptr addrspace(3) %ptr) {
; GFX6-LABEL: load_lds_v3i32_align4:
; GFX6: ; %bb.0:
; GFX6-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; GFX6-NEXT: v_add_i32_e32 v1, vcc, 4, v0
-; GFX6-NEXT: v_add_i32_e32 v2, vcc, 8, v0
; GFX6-NEXT: s_mov_b32 m0, -1
-; GFX6-NEXT: ds_read_b32 v2, v2
-; GFX6-NEXT: ds_read_b32 v0, v0
+; GFX6-NEXT: v_add_i32_e32 v1, vcc, 4, v0
+; GFX6-NEXT: ds_read_b32 v3, v0
+; GFX6-NEXT: v_add_i32_e32 v0, vcc, 8, v0
; GFX6-NEXT: ds_read_b32 v1, v1
+; GFX6-NEXT: ds_read_b32 v2, v0
+; GFX6-NEXT: s_waitcnt lgkmcnt(2)
+; GFX6-NEXT: v_mov_b32_e32 v0, v3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: s_setpc_b64 s[30:31]
;
diff --git a/llvm/test/CodeGen/AMDGPU/local-memory.amdgcn.ll b/llvm/test/CodeGen/AMDGPU/local-memory.amdgcn.ll
index 2444b2d59a6f8..2dd0068ef0f3d 100644
--- a/llvm/test/CodeGen/AMDGPU/local-memory.amdgcn.ll
+++ b/llvm/test/CodeGen/AMDGPU/local-memory.amdgcn.ll
@@ -54,8 +54,8 @@ define amdgpu_kernel void @local_memory_two_objects(ptr addrspace(1) %out) #0 {
; SI-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x9
; SI-NEXT: s_waitcnt lgkmcnt(0)
; SI-NEXT: s_barrier
-; SI-NEXT: v_sub_i32_e32 v2, vcc, 28, v1
; SI-NEXT: ds_read_b32 v0, v0
+; SI-NEXT: v_sub_i32_e32 v2, vcc, 28, v1
; SI-NEXT: ds_read_b32 v3, v2
; SI-NEXT: s_mov_b32 s3, 0xf000
; SI-NEXT: s_mov_b32 s2, 0
diff --git a/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa-memcpy.ll b/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa-memcpy.ll
index 4ab05c2923fdb..abc0ac46f470c 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa-memcpy.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa-memcpy.ll
@@ -22,14 +22,14 @@ define protected amdgpu_kernel void @test(ptr addrspace(1) nocapture %ptr.coerce
; GCN-NEXT: v_mov_b32_e32 v0, 2
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: ds_write_b8 v1, v0
-; GCN-NEXT: ds_read_u8 v2, v1 offset:2
; GCN-NEXT: ds_read_u16 v3, v1
+; GCN-NEXT: ds_read_u8 v2, v1 offset:2
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: ds_write_b8 v1, v2 offset:6
-; GCN-NEXT: ds_write_b16 v1, v3 offset:4
; GCN-NEXT: v_cmp_eq_u32_sdwa s[2:3], v3, v0 src0_sel:BYTE_0 src1_sel:DWORD
; GCN-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[2:3]
+; GCN-NEXT: ds_write_b8 v1, v2 offset:6
+; GCN-NEXT: ds_write_b16 v1, v3 offset:4
; GCN-NEXT: global_store_byte v1, v0, s[0:1]
; GCN-NEXT: s_endpgm
; CHECK-LABEL: define protected amdgpu_kernel void @test(
diff --git a/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa.ll b/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa.ll
index 24c1bfb8d50f0..484c0c442b9bf 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-lds-struct-aa.ll
@@ -35,11 +35,11 @@ define amdgpu_kernel void @no_clobber_ds_load_stores_x2(ptr addrspace(1) %arg, i
; GCN-NEXT: s_lshl_b32 s0, s0, 2
; GCN-NEXT: v_mov_b32_e32 v0, s0
; GCN-NEXT: ds_write_b32 v1, v2 offset:256
-; GCN-NEXT: ds_read_b32 v2, v0
+; GCN-NEXT: ds_read_b32 v3, v0
; GCN-NEXT: ds_read_b32 v0, v0 offset:256
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_u32_e32 v0, v2, v0
+; GCN-NEXT: v_add_u32_e32 v0, v3, v0
; GCN-NEXT: global_store_dword v1, v0, s[0:1]
; GCN-NEXT: s_endpgm
bb:
@@ -75,22 +75,22 @@ define amdgpu_kernel void @no_clobber_ds_load_stores_x3(ptr addrspace(1) %arg, i
; GCN-LABEL: no_clobber_ds_load_stores_x3:
; GCN: ; %bb.0: ; %bb
; GCN-NEXT: s_load_dword s0, s[4:5], 0x2c
+; GCN-NEXT: v_mov_b32_e32 v0, 1
; GCN-NEXT: v_mov_b32_e32 v1, 0
; GCN-NEXT: v_mov_b32_e32 v2, 2
-; GCN-NEXT: v_mov_b32_e32 v0, 1
-; GCN-NEXT: ds_write_b32 v1, v2 offset:256
+; GCN-NEXT: ds_write_b32 v1, v0
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: s_lshl_b32 s0, s0, 2
-; GCN-NEXT: v_mov_b32_e32 v2, 3
-; GCN-NEXT: ds_write_b32 v1, v0
; GCN-NEXT: v_mov_b32_e32 v0, s0
-; GCN-NEXT: ds_write_b32 v1, v2 offset:512
-; GCN-NEXT: ds_read_b32 v2, v0
-; GCN-NEXT: ds_read_b32 v3, v0 offset:256
+; GCN-NEXT: ds_write_b32 v1, v2 offset:256
+; GCN-NEXT: v_mov_b32_e32 v4, 3
+; GCN-NEXT: ds_read_b32 v3, v0
+; GCN-NEXT: ds_read_b32 v2, v0 offset:256
+; GCN-NEXT: ds_write_b32 v1, v4 offset:512
; GCN-NEXT: ds_read_b32 v0, v0 offset:512
; GCN-NEXT: s_load_dwordx2 s[0:1], s[4:5], 0x24
; GCN-NEXT: s_waitcnt lgkmcnt(0)
-; GCN-NEXT: v_add_u32_e32 v2, v2, v3
+; GCN-NEXT: v_add_u32_e32 v2, v3, v2
; GCN-NEXT: v_add_u32_e32 v0, v2, v0
; GCN-NEXT: global_store_dword v1, v0, s[0:1]
; GCN-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll b/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
index 5b7c36559a366..204722c8d84df 100644
--- a/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
+++ b/llvm/test/CodeGen/AMDGPU/memcpy-libcall.ll
@@ -268,11 +268,11 @@ define amdgpu_kernel void @memcpy_p0_p3_minsize(ptr %generic) #0 {
; CHECK-NEXT: ds_read2_b64 v[0:3], v16 offset1:1
; CHECK-NEXT: ds_read2_b64 v[4:7], v16 offset0:2 offset1:3
; CHECK-NEXT: ds_read2_b64 v[8:11], v16 offset0:4 offset1:5
-; CHECK-NEXT: ds_read2_b64 v[12:15], v16 offset0:6 offset1:7
; CHECK-NEXT: s_add_u32 flat_scratch_lo, s12, s17
-; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v21, s1
+; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
+; CHECK-NEXT: ds_read2_b64 v[12:15], v16 offset0:6 offset1:7
; CHECK-NEXT: v_mov_b32_e32 v20, s0
; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[0:3]
; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[4:7] offset:16
@@ -281,8 +281,8 @@ define amdgpu_kernel void @memcpy_p0_p3_minsize(ptr %generic) #0 {
; CHECK-NEXT: ds_read2_b64 v[4:7], v16 offset0:10 offset1:11
; CHECK-NEXT: ds_read_b128 v[8:11], v16 offset:96
; CHECK-NEXT: ds_read_b128 v[16:19], v16 offset:112
-; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[12:15] offset:48
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[12:15] offset:48
; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[0:3] offset:64
; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[4:7] offset:80
; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[8:11] offset:96
@@ -556,11 +556,11 @@ define amdgpu_kernel void @memcpy_p0_p3_optsize(ptr %generic) #1 {
; CHECK-NEXT: ds_read2_b64 v[0:3], v16 offset1:1
; CHECK-NEXT: ds_read2_b64 v[4:7], v16 offset0:2 offset1:3
; CHECK-NEXT: ds_read2_b64 v[8:11], v16 offset0:4 offset1:5
-; CHECK-NEXT: ds_read2_b64 v[12:15], v16 offset0:6 offset1:7
; CHECK-NEXT: s_add_u32 flat_scratch_lo, s12, s17
-; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: v_mov_b32_e32 v21, s1
+; CHECK-NEXT: s_addc_u32 flat_scratch_hi, s13, 0
+; CHECK-NEXT: ds_read2_b64 v[12:15], v16 offset0:6 offset1:7
; CHECK-NEXT: v_mov_b32_e32 v20, s0
; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[0:3]
; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[4:7] offset:16
@@ -569,8 +569,8 @@ define amdgpu_kernel void @memcpy_p0_p3_optsize(ptr %generic) #1 {
; CHECK-NEXT: ds_read2_b64 v[4:7], v16 offset0:10 offset1:11
; CHECK-NEXT: ds_read_b128 v[8:11], v16 offset:96
; CHECK-NEXT: ds_read_b128 v[16:19], v16 offset:112
-; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[12:15] offset:48
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
+; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[12:15] offset:48
; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[0:3] offset:64
; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[4:7] offset:80
; CHECK-NEXT: flat_store_dwordx4 v[20:21], v[8:11] offset:96
diff --git a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
index c60642e2cc4d8..247b158807e93 100644
--- a/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
+++ b/llvm/test/CodeGen/AMDGPU/memintrinsic-unroll.ll
@@ -16849,6 +16849,7 @@ define void @memset_p3_sz2048(ptr addrspace(3) %dst) {
; CHECK-NEXT: s_add_u32 s4, s4, 0x100
; CHECK-NEXT: s_addc_u32 s5, s5, 0
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:240
+; CHECK-NEXT: v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:224
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:208
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:192
@@ -16859,7 +16860,6 @@ define void @memset_p3_sz2048(ptr addrspace(3) %dst) {
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:112
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:96
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:80
-; CHECK-NEXT: v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:64
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:48
; CHECK-NEXT: ds_write_b128 v0, v[1:4] offset:32
@@ -16883,6 +16883,7 @@ define void @memset_p3_sz2048(ptr addrspace(3) %dst) {
; ALIGNED-NEXT: s_add_u32 s4, s4, 0x100
; ALIGNED-NEXT: s_addc_u32 s5, s5, 0
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:255
+; ALIGNED-NEXT: v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:254
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:253
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:252
@@ -17133,7 +17134,6 @@ define void @memset_p3_sz2048(ptr addrspace(3) %dst) {
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:7
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:6
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:5
-; ALIGNED-NEXT: v_cmp_gt_u64_e64 s6, 0x800, s[4:5]
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:4
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:3
; ALIGNED-NEXT: ds_write_b8 v0, v1 offset:2
@@ -17164,9 +17164,9 @@ define void @memset_p3_sz2048(ptr addrspace(3) %dst) {
; UNROLL3-NEXT: s_add_u32 s4, s4, 48
; UNROLL3-NEXT: s_addc_u32 s5, s5, 0
; UNROLL3-NEXT: ds_write_b128 v5, v[1:4] offset:16
+; UNROLL3-NEXT: v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
; UNROLL3-NEXT: ds_write_b128 v5, v[1:4]
; UNROLL3-NEXT: ds_write_b128 v5, v[1:4] offset:32
-; UNROLL3-NEXT: v_cmp_gt_u64_e64 s6, 0x7e0, s[4:5]
; UNROLL3-NEXT: v_add_nc_u32_e32 v5, 48, v5
; UNROLL3-NEXT: s_and_b32 vcc_lo, exec_lo, s6
; UNROLL3-NEXT: s_cbranch_vccnz .LBB12_1
diff --git a/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
index 01b7f40f6256f..7105e66b58e93 100644
--- a/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memmove-param-combinations.ll
@@ -485,14 +485,14 @@ define void @memmove_p0_p3_sz31_align_1_1(ptr addrspace(0) align 1 %dst, ptr add
; CHECK-LABEL: memmove_p0_p3_sz31_align_1_1:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: ds_read_b32 v8, v2 offset:24
; CHECK-NEXT: ds_read_u8 v9, v2 offset:30
; CHECK-NEXT: ds_read_u16 v10, v2 offset:28
; CHECK-NEXT: ds_read_b64 v[6:7], v2 offset:16
+; CHECK-NEXT: ds_read_b32 v8, v2 offset:24
; CHECK-NEXT: ds_read_b128 v[2:5], v2
-; CHECK-NEXT: s_waitcnt lgkmcnt(3)
+; CHECK-NEXT: s_waitcnt lgkmcnt(4)
; CHECK-NEXT: flat_store_byte v[0:1], v9 offset:30
-; CHECK-NEXT: s_waitcnt lgkmcnt(3)
+; CHECK-NEXT: s_waitcnt lgkmcnt(4)
; CHECK-NEXT: flat_store_short v[0:1], v10 offset:28
; CHECK-NEXT: s_waitcnt lgkmcnt(3)
; CHECK-NEXT: flat_store_dwordx3 v[0:1], v[6:8] offset:16
@@ -540,14 +540,14 @@ define void @memmove_p0_p3_sz31_align_2_2(ptr addrspace(0) align 2 %dst, ptr add
; CHECK-LABEL: memmove_p0_p3_sz31_align_2_2:
; CHECK: ; %bb.0: ; %entry
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
-; CHECK-NEXT: ds_read_b32 v8, v2 offset:24
; CHECK-NEXT: ds_read_u8 v9, v2 offset:30
; CHECK-NEXT: ds_read_u16 v10, v2 offset:28
; CHECK-NEXT: ds_read_b64 v[6:7], v2 offset:16
+; CHECK-NEXT: ds_read_b32 v8, v2 offset:24
; CHECK-NEXT: ds_read_b128 v[2:5], v2
-; CHECK-NEXT: s_waitcnt lgkmcnt(3)
+; CHECK-NEXT: s_waitcnt lgkmcnt(4)
; CHECK-NEXT: flat_store_byte v[0:1], v9 offset:30
-; CHECK-NEXT: s_waitcnt lgkmcnt(3)
+; CHECK-NEXT: s_waitcnt lgkmcnt(4)
; CHECK-NEXT: flat_store_short v[0:1], v10 offset:28
; CHECK-NEXT: s_waitcnt lgkmcnt(3)
; CHECK-NEXT: flat_store_dwordx3 v[0:1], v[6:8] offset:16
@@ -1755,16 +1755,17 @@ define void @memmove_p1_p3_sz31_align_8_8(ptr addrspace(1) align 8 %dst, ptr add
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: ds_read2_b64 v[3:6], v2 offset1:1
; CHECK-NEXT: ds_read_b32 v7, v2 offset:16
-; CHECK-NEXT: ds_read_u8 v10, v2 offset:30
; CHECK-NEXT: ds_read2_b32 v[8:9], v2 offset0:5 offset1:6
-; CHECK-NEXT: ds_read_u16 v2, v2 offset:28
+; CHECK-NEXT: ds_read_u16 v10, v2 offset:28
+; CHECK-NEXT: ds_read_u8 v2, v2 offset:30
; CHECK-NEXT: s_waitcnt lgkmcnt(4)
; CHECK-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
-; CHECK-NEXT: s_waitcnt lgkmcnt(1)
+; CHECK-NEXT: s_waitcnt lgkmcnt(2)
; CHECK-NEXT: global_store_dwordx3 v[0:1], v[7:9], off offset:16
+; CHECK-NEXT: s_waitcnt lgkmcnt(1)
+; CHECK-NEXT: global_store_short v[0:1], v10, off offset:28
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_short v[0:1], v2, off offset:28
-; CHECK-NEXT: global_store_byte v[0:1], v10, off offset:30
+; CHECK-NEXT: global_store_byte v[0:1], v2, off offset:30
; CHECK-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memmove.p1.p3.i64(ptr addrspace(1) noundef nonnull align 8 %dst, ptr addrspace(3) noundef nonnull align 8 %src, i64 31, i1 false)
@@ -1806,16 +1807,17 @@ define void @memmove_p1_p3_sz31_align_16_16(ptr addrspace(1) align 16 %dst, ptr
; CHECK-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CHECK-NEXT: ds_read_b128 v[3:6], v2
; CHECK-NEXT: ds_read_b32 v7, v2 offset:16
-; CHECK-NEXT: ds_read_u8 v10, v2 offset:30
; CHECK-NEXT: ds_read2_b32 v[8:9], v2 offset0:5 offset1:6
-; CHECK-NEXT: ds_read_u16 v2, v2 offset:28
+; CHECK-NEXT: ds_read_u16 v10, v2 offset:28
+; CHECK-NEXT: ds_read_u8 v2, v2 offset:30
; CHECK-NEXT: s_waitcnt lgkmcnt(4)
; CHECK-NEXT: global_store_dwordx4 v[0:1], v[3:6], off
-; CHECK-NEXT: s_waitcnt lgkmcnt(1)
+; CHECK-NEXT: s_waitcnt lgkmcnt(2)
; CHECK-NEXT: global_store_dwordx3 v[0:1], v[7:9], off offset:16
+; CHECK-NEXT: s_waitcnt lgkmcnt(1)
+; CHECK-NEXT: global_store_short v[0:1], v10, off offset:28
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
-; CHECK-NEXT: global_store_short v[0:1], v2, off offset:28
-; CHECK-NEXT: global_store_byte v[0:1], v10, off offset:30
+; CHECK-NEXT: global_store_byte v[0:1], v2, off offset:30
; CHECK-NEXT: s_setpc_b64 s[30:31]
entry:
tail call void @llvm.memmove.p1.p3.i64(ptr addrspace(1) noundef nonnull align 16 %dst, ptr addrspace(3) noundef nonnull align 16 %src, i64 31, i1 false)
@@ -2525,9 +2527,9 @@ define void @memmove_p3_p0_sz31_align_16_16(ptr addrspace(3) align 16 %dst, ptr
; CHECK-NEXT: flat_load_dword v1, v[1:2] offset:24
; CHECK-NEXT: ds_write_b8 v0, v7 offset:30
; CHECK-NEXT: ds_write_b16 v0, v9 offset:28
-; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(2)
-; CHECK-NEXT: ds_write_b32 v0, v1 offset:24
; CHECK-NEXT: ds_write_b128 v0, v[3:6]
+; CHECK-NEXT: s_waitcnt vmcnt(0) lgkmcnt(3)
+; CHECK-NEXT: ds_write_b32 v0, v1 offset:24
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
; CHECK-NEXT: s_setpc_b64 s[30:31]
entry:
@@ -3269,11 +3271,12 @@ define void @memmove_p3_p5_sz31_align_1_1(ptr addrspace(3) align 1 %dst, ptr add
; CHECK-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
; CHECK-NEXT: buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
; CHECK-NEXT: buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT: s_waitcnt vmcnt(8)
+; CHECK-NEXT: ds_write_b8 v0, v8 offset:30
; CHECK-NEXT: s_waitcnt vmcnt(4)
; CHECK-NEXT: ds_write_b32 v0, v9 offset:24
; CHECK-NEXT: s_waitcnt vmcnt(3)
; CHECK-NEXT: ds_write_b16 v0, v10 offset:28
-; CHECK-NEXT: ds_write_b8 v0, v8 offset:30
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -3341,11 +3344,12 @@ define void @memmove_p3_p5_sz31_align_2_2(ptr addrspace(3) align 2 %dst, ptr add
; CHECK-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
; CHECK-NEXT: buffer_load_dword v6, v1, s[0:3], 0 offen offset:16
; CHECK-NEXT: buffer_load_dword v7, v1, s[0:3], 0 offen offset:20
+; CHECK-NEXT: s_waitcnt vmcnt(8)
+; CHECK-NEXT: ds_write_b8 v0, v8 offset:30
; CHECK-NEXT: s_waitcnt vmcnt(4)
; CHECK-NEXT: ds_write_b32 v0, v9 offset:24
; CHECK-NEXT: s_waitcnt vmcnt(3)
; CHECK-NEXT: ds_write_b16 v0, v10 offset:28
-; CHECK-NEXT: ds_write_b8 v0, v8 offset:30
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
; CHECK-NEXT: s_waitcnt vmcnt(0)
@@ -3486,12 +3490,14 @@ define void @memmove_p3_p5_sz31_align_16_16(ptr addrspace(3) align 16 %dst, ptr
; CHECK-NEXT: buffer_load_ushort v10, v1, s[0:3], 0 offen offset:28
; CHECK-NEXT: buffer_load_dword v4, v1, s[0:3], 0 offen offset:8
; CHECK-NEXT: buffer_load_dword v5, v1, s[0:3], 0 offen offset:12
+; CHECK-NEXT: s_waitcnt vmcnt(8)
+; CHECK-NEXT: ds_write_b8 v0, v6 offset:30
+; CHECK-NEXT: s_waitcnt vmcnt(6)
+; CHECK-NEXT: ds_write_b32 v0, v7 offset:16
; CHECK-NEXT: s_waitcnt vmcnt(4)
; CHECK-NEXT: ds_write2_b32 v0, v8, v9 offset0:5 offset1:6
-; CHECK-NEXT: ds_write_b32 v0, v7 offset:16
; CHECK-NEXT: s_waitcnt vmcnt(2)
; CHECK-NEXT: ds_write_b16 v0, v10 offset:28
-; CHECK-NEXT: ds_write_b8 v0, v6 offset:30
; CHECK-NEXT: s_waitcnt vmcnt(0)
; CHECK-NEXT: ds_write_b128 v0, v[2:5]
; CHECK-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
index 990a986ffab75..2238b04b39c2d 100644
--- a/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
+++ b/llvm/test/CodeGen/AMDGPU/memset-param-combinations.ll
@@ -1184,15 +1184,15 @@ define void @memset_p3_sz1055_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-SDAG-NEXT: ; %bb.2: ; %static-memset-post-expansion
; GFX942-SDAG-NEXT: s_mov_b32 s0, 0x4040404
; GFX942-SDAG-NEXT: v_add_u32_e32 v2, 0x400, v0
-; GFX942-SDAG-NEXT: v_add_u32_e32 v3, 0x408, v0
; GFX942-SDAG-NEXT: v_perm_b32 v4, v1, v1, s0
-; GFX942-SDAG-NEXT: ds_write2_b32 v3, v4, v4 offset1:1
; GFX942-SDAG-NEXT: ds_write2_b32 v2, v4, v4 offset1:1
; GFX942-SDAG-NEXT: v_add_u32_e32 v2, 0x410, v0
; GFX942-SDAG-NEXT: ds_write2_b32 v2, v4, v4 offset1:1
-; GFX942-SDAG-NEXT: ds_write_b32 v0, v4 offset:1048
; GFX942-SDAG-NEXT: v_lshlrev_b16_e32 v2, 8, v1
+; GFX942-SDAG-NEXT: v_add_u32_e32 v3, 0x408, v0
; GFX942-SDAG-NEXT: v_or_b32_sdwa v2, v1, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
+; GFX942-SDAG-NEXT: ds_write2_b32 v3, v4, v4 offset1:1
+; GFX942-SDAG-NEXT: ds_write_b32 v0, v4 offset:1048
; GFX942-SDAG-NEXT: ds_write_b16 v0, v2 offset:1052
; GFX942-SDAG-NEXT: ds_write_b8 v0, v1 offset:1054
; GFX942-SDAG-NEXT: s_waitcnt lgkmcnt(0)
@@ -1217,6 +1217,8 @@ define void @memset_p3_sz1055_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT: v_add_co_u32_e32 v8, vcc, 0x100, v8
; GFX942-GISEL-NEXT: ds_write2_b64 v10, v[2:3], v[4:5] offset1:1
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
; GFX942-GISEL-NEXT: ds_write2_b64 v10, v[2:3], v[4:5] offset0:2 offset1:3
; GFX942-GISEL-NEXT: ds_write2_b64 v10, v[2:3], v[4:5] offset0:4 offset1:5
; GFX942-GISEL-NEXT: ds_write2_b64 v10, v[2:3], v[4:5] offset0:6 offset1:7
@@ -1232,7 +1234,6 @@ define void @memset_p3_sz1055_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-GISEL-NEXT: ds_write2_b64 v10, v[2:3], v[4:5] offset0:26 offset1:27
; GFX942-GISEL-NEXT: ds_write2_b64 v10, v[2:3], v[4:5] offset0:28 offset1:29
; GFX942-GISEL-NEXT: ds_write2_b64 v10, v[2:3], v[4:5] offset0:30 offset1:31
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
; GFX942-GISEL-NEXT: v_add_u32_e32 v10, 0x100, v10
; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[6:7]
; GFX942-GISEL-NEXT: s_cbranch_vccnz .LBB8_1
@@ -1363,6 +1364,8 @@ define void @memset_p3_sz2048_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-GISEL-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX942-GISEL-NEXT: v_add_co_u32_e32 v8, vcc, 0x100, v8
; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset1:1
+; GFX942-GISEL-NEXT: s_nop 0
+; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:2 offset1:3
; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:4 offset1:5
; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:6 offset1:7
@@ -1378,7 +1381,6 @@ define void @memset_p3_sz2048_align_4_varsetval(ptr addrspace(3) align 4 %dst, i
; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:26 offset1:27
; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:28 offset1:29
; GFX942-GISEL-NEXT: ds_write2_b64 v0, v[2:3], v[4:5] offset0:30 offset1:31
-; GFX942-GISEL-NEXT: v_addc_co_u32_e32 v9, vcc, 0, v9, vcc
; GFX942-GISEL-NEXT: v_add_u32_e32 v0, 0x100, v0
; GFX942-GISEL-NEXT: v_cmp_lt_u64_e32 vcc, v[8:9], v[6:7]
; GFX942-GISEL-NEXT: s_cbranch_vccnz .LBB9_1
diff --git a/llvm/test/CodeGen/AMDGPU/misched-remat-revert.ll b/llvm/test/CodeGen/AMDGPU/misched-remat-revert.ll
index 30f149de74e60..c4f5f9c0cf8a9 100644
--- a/llvm/test/CodeGen/AMDGPU/misched-remat-revert.ll
+++ b/llvm/test/CodeGen/AMDGPU/misched-remat-revert.ll
@@ -1,6 +1,8 @@
-; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -O3 -debug-only=machine-scheduler 2>&1 < %s | FileCheck -check-prefix=DEBUG %s
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx942 -O3 -debug-only=machine-scheduler 2>&1 < %s
; REQUIRES: asserts
+; This testcase only checks it compiles successfully.
+
; This testcase hit a situation where reverting scheduling after the scheduler's
; rematerialization stage would cause incoherent MI and slot orders, hitting an
; assert later on in the RP trackers. This is fixed by ensuring that all MIs that
diff --git a/llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll b/llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll
index 05ff5c8bb0b3a..42dd4fa24bd1a 100644
--- a/llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll
+++ b/llvm/test/CodeGen/AMDGPU/module-lds-false-sharing.ll
@@ -48,9 +48,9 @@ define void @nonkernel() {
; G_GFX9-LABEL: nonkernel:
; G_GFX9: ; %bb.0:
; G_GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; G_GFX9-NEXT: v_mov_b32_e32 v0, 0
; G_GFX9-NEXT: v_mov_b32_e32 v2, 0
; G_GFX9-NEXT: v_mov_b32_e32 v3, 8
-; G_GFX9-NEXT: v_mov_b32_e32 v0, 0
; G_GFX9-NEXT: v_mov_b32_e32 v1, 0
; G_GFX9-NEXT: ds_write_b32 v3, v2
; G_GFX9-NEXT: ds_write_b64 v2, v[0:1]
@@ -60,9 +60,9 @@ define void @nonkernel() {
; G_GFX10-LABEL: nonkernel:
; G_GFX10: ; %bb.0:
; G_GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
+; G_GFX10-NEXT: v_mov_b32_e32 v0, 0
; G_GFX10-NEXT: v_mov_b32_e32 v2, 0
; G_GFX10-NEXT: v_mov_b32_e32 v3, 8
-; G_GFX10-NEXT: v_mov_b32_e32 v0, 0
; G_GFX10-NEXT: v_mov_b32_e32 v1, 0
; G_GFX10-NEXT: ds_write_b32 v3, v2
; G_GFX10-NEXT: ds_write_b64 v2, v[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
index b1e304e79348e..d5fad3602d859 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-op-sel.ll
@@ -766,12 +766,12 @@ define amdgpu_kernel void @mix_elt_types_op_sel(ptr addrspace(1) %out, ptr addrs
; GCN-NEXT: ds_read_b32 v0, v0 offset:8
; GCN-NEXT: s_load_dwordx2 s[2:3], s[0:1], 0x24
; GCN-NEXT: ; kill: killed $vgpr0_vgpr1
+; GCN-NEXT: ; kill: killed $vgpr0_vgpr1
; GCN-NEXT: s_waitcnt lgkmcnt(0)
; GCN-NEXT: global_load_ushort v3, v[0:1], off glc
; GCN-NEXT: s_waitcnt vmcnt(0)
; GCN-NEXT: global_load_ushort v3, v[0:1], off glc
; GCN-NEXT: s_waitcnt vmcnt(0)
-; GCN-NEXT: ; kill: killed $vgpr0_vgpr1
; GCN-NEXT: v_pk_add_f16 v0, v0, 2.0 op_sel_hi:[1,0]
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: v_pk_fma_f16 v0, v1, v2, v0 op_sel:[0,0,1] op_sel_hi:[1,1,0]
diff --git a/llvm/test/CodeGen/AMDGPU/postra-bundle-memops.mir b/llvm/test/CodeGen/AMDGPU/postra-bundle-memops.mir
index 40936c5d23bf0..83d8f5241a58e 100644
--- a/llvm/test/CodeGen/AMDGPU/postra-bundle-memops.mir
+++ b/llvm/test/CodeGen/AMDGPU/postra-bundle-memops.mir
@@ -35,14 +35,10 @@ body: |
; GCN-NEXT: $vgpr0 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, implicit $exec
; GCN-NEXT: S_NOP 0
; GCN-NEXT: GLOBAL_STORE_DWORD undef $vgpr3_vgpr4, $vgpr0, 4, 0, implicit $exec
- ; GCN-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit $vgpr0, implicit $exec, implicit $vgpr1 {
- ; GCN-NEXT: $vgpr2 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
- ; GCN-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec
- ; GCN-NEXT: }
- ; GCN-NEXT: BUNDLE implicit $vgpr0, implicit $vgpr2, implicit killed $m0, implicit $exec, implicit $vgpr3 {
- ; GCN-NEXT: DS_WRITE_B32_gfx9 $vgpr0, $vgpr2, 0, 0, implicit killed $m0, implicit $exec
- ; GCN-NEXT: DS_WRITE_B32_gfx9 $vgpr0, $vgpr3, 4, 0, implicit killed $m0, implicit $exec
- ; GCN-NEXT: }
+ ; GCN-NEXT: $vgpr2 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
+ ; GCN-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec
+ ; GCN-NEXT: DS_WRITE_B32_gfx9 $vgpr0, $vgpr2, 0, 0, implicit killed $m0, implicit $exec
+ ; GCN-NEXT: DS_WRITE_B32_gfx9 $vgpr0, $vgpr3, 4, 0, implicit killed $m0, implicit $exec
; GCN-NEXT: S_NOP 0
; GCN-NEXT: BUNDLE implicit-def $sgpr2, implicit-def $sgpr3, implicit undef $sgpr0_sgpr1, implicit undef $sgpr10 {
; GCN-NEXT: $sgpr2 = S_LOAD_DWORD_IMM undef $sgpr0_sgpr1, 0, 0
@@ -68,10 +64,8 @@ body: |
; GCN-NEXT: $sgpr64_sgpr65_sgpr66_sgpr67_sgpr68_sgpr69_sgpr70_sgpr71 = S_LOAD_DWORDX8_IMM undef $sgpr10_sgpr11, 464, 0
; GCN-NEXT: $sgpr68_sgpr69_sgpr70_sgpr71_sgpr72_sgpr73_sgpr74_sgpr75 = S_LOAD_DWORDX8_IMM undef $sgpr10_sgpr11, 128, 0
; GCN-NEXT: S_NOP 0
- ; GCN-NEXT: BUNDLE implicit-def $vgpr2, implicit-def $vgpr3, implicit $vgpr0, implicit $exec, implicit $vgpr1 {
- ; GCN-NEXT: $vgpr2 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
- ; GCN-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec
- ; GCN-NEXT: }
+ ; GCN-NEXT: $vgpr2 = DS_READ_B32_gfx9 $vgpr0, 0, 0, implicit $exec
+ ; GCN-NEXT: $vgpr3 = DS_READ_B32_gfx9 $vgpr1, 0, 0, implicit $exec
$vgpr0 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, implicit $exec
S_NOP 0
$vgpr0 = GLOBAL_LOAD_DWORD undef $vgpr3_vgpr4, 0, 0, implicit $exec
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
index aa0d1fe45e9a8..92f57266d1b84 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
+++ b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-ilp-metric-spills.mir
@@ -14,12 +14,7 @@
# GCN: S_ENDPGM
# GCN-GCNTRACKER-LABEL: name: no_sched_metric_due_to_spills
-# GCN-GCNTRACKER: SI_SPILL_V32_SAVE
-# GCN-GCNTRACKER: SI_SPILL_V32_SAVE
-# GCN-GCNTRACKER: SI_SPILL_V32_SAVE
-# GCN-GCNTRACKER: SI_SPILL_V32_SAVE
-# GCN-GCNTRACKER: SI_SPILL_V32_SAVE
-# GCN-GCNTRACKER: SI_SPILL_V32_SAVE
+# GCN-GCNTRACKER-NOT: SI_SPILL_
# GCN-GCNTRACKER: S_ENDPGM
# When using the GCN Trackers, the scheduler is able to acieve desired occupancy without running high-RP-reschedule stage. However, the RP is still high,
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-limit2.ll b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-limit2.ll
index 22ea449c535ac..dc5d065dbe146 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-regpressure-limit2.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-regpressure-limit2.ll
@@ -18,8 +18,8 @@
; VI-MINREG: NumVgprs: {{[1-3][0-9]$}}
; stores may alias loads
-; VI-MAXOCC: NumSgprs: {{[1-3][0-9]$}}
-; VI-MAXOCC: NumVgprs: {{[1-6][0-9]$}}
+; VI-MAXOCC: NumSgprs: {{[1-3]?[0-9]$}}
+; VI-MAXOCC: NumVgprs: {{[1-6]?[0-9]$}}
define amdgpu_kernel void @load_fma_store(ptr addrspace(3) nocapture readonly %in_arg, ptr addrspace(1) nocapture %out_arg) {
bb:
diff --git a/llvm/test/CodeGen/AMDGPU/schedule-relaxed-occupancy.ll b/llvm/test/CodeGen/AMDGPU/schedule-relaxed-occupancy.ll
index 7b8eba1091b48..2ce4f1dbad46d 100644
--- a/llvm/test/CodeGen/AMDGPU/schedule-relaxed-occupancy.ll
+++ b/llvm/test/CodeGen/AMDGPU/schedule-relaxed-occupancy.ll
@@ -7,16 +7,16 @@
; Using -amgpu-schedule-relaxed-occupancy allows scheduler to produce better ILP by further relaxing occupancy target
; CHECK-LABEL: {{^}}load_fma_store:
-; OCC: NumVgprs: 24
-; OCC-GCNTRACKER: NumVgprs: 26
-; RELAX: NumVgprs: 64
+; OCC: NumVgprs: 20
+; OCC-GCNTRACKER: NumVgprs: 22
+; RELAX: NumVgprs: 60
; RELAX-GCNTRACKER: NumVgprs: 60
-; OCC: NumVGPRsForWavesPerEU: 24
-; OCC-GCNTRACKER: NumVGPRsForWavesPerEU: 26
-; RELAX: NumVGPRsForWavesPerEU: 64
+; OCC: NumVGPRsForWavesPerEU: 20
+; OCC-GCNTRACKER: NumVGPRsForWavesPerEU: 22
+; RELAX: NumVGPRsForWavesPerEU: 60
; RELAX-GCNTRACKER: NumVGPRsForWavesPerEU: 60
; OCC: Occupancy: 10
-; OCC-GCNTRACKER: Occupancy: 9
+; OCC-GCNTRACKER: Occupancy: 10
; RELAX: Occupancy: 4
; RELAX-GCNTRACKER: Occupancy: 4
diff --git a/llvm/test/CodeGen/AMDGPU/store-local.128.ll b/llvm/test/CodeGen/AMDGPU/store-local.128.ll
index a4e23ae87614f..185e9139e4f21 100644
--- a/llvm/test/CodeGen/AMDGPU/store-local.128.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-local.128.ll
@@ -84,11 +84,8 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s6
; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: ds_write_b8 v0, v1 offset:12
; GFX9-NEXT: ds_write_b8_d16_hi v0, v1 offset:14
-; GFX9-NEXT: ds_write_b8 v0, v2 offset:8
-; GFX9-NEXT: ds_write_b8_d16_hi v0, v2 offset:10
; GFX9-NEXT: v_mov_b32_e32 v1, s1
; GFX9-NEXT: ds_write_b8 v0, v1 offset:4
; GFX9-NEXT: ds_write_b8_d16_hi v0, v1 offset:6
@@ -101,6 +98,7 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX9-NEXT: ds_write_b8 v0, v1 offset:13
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: s_lshr_b32 s3, s2, 8
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: ds_write_b8 v0, v1 offset:15
; GFX9-NEXT: v_mov_b32_e32 v1, s3
; GFX9-NEXT: s_lshr_b32 s2, s2, 24
@@ -118,6 +116,8 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX9-NEXT: s_lshr_b32 s0, s0, 24
; GFX9-NEXT: ds_write_b8 v0, v1 offset:1
; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: ds_write_b8 v0, v2 offset:8
+; GFX9-NEXT: ds_write_b8_d16_hi v0, v2 offset:10
; GFX9-NEXT: ds_write_b8 v0, v1 offset:3
; GFX9-NEXT: s_endpgm
;
@@ -129,9 +129,7 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, s6
; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: ds_write_b8 v0, v1 offset:12
-; GFX7-NEXT: ds_write_b8 v0, v2 offset:8
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: ds_write_b8 v0, v1 offset:4
; GFX7-NEXT: v_mov_b32_e32 v1, s0
@@ -148,6 +146,7 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX7-NEXT: ds_write_b8 v0, v1 offset:14
; GFX7-NEXT: v_mov_b32_e32 v1, s3
; GFX7-NEXT: s_lshr_b32 s3, s2, 24
+; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: ds_write_b8 v0, v1 offset:9
; GFX7-NEXT: v_mov_b32_e32 v1, s3
; GFX7-NEXT: s_lshr_b32 s2, s2, 16
@@ -171,6 +170,7 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX7-NEXT: s_lshr_b32 s0, s0, 16
; GFX7-NEXT: ds_write_b8 v0, v1 offset:3
; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: ds_write_b8 v0, v2 offset:8
; GFX7-NEXT: ds_write_b8 v0, v1 offset:2
; GFX7-NEXT: s_endpgm
;
@@ -182,9 +182,7 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, s6
; GFX6-NEXT: v_mov_b32_e32 v1, s3
-; GFX6-NEXT: v_mov_b32_e32 v2, s2
; GFX6-NEXT: ds_write_b8 v0, v1 offset:12
-; GFX6-NEXT: ds_write_b8 v0, v2 offset:8
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: ds_write_b8 v0, v1 offset:4
; GFX6-NEXT: v_mov_b32_e32 v1, s0
@@ -201,6 +199,7 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX6-NEXT: ds_write_b8 v0, v1 offset:14
; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: s_lshr_b32 s3, s2, 24
+; GFX6-NEXT: v_mov_b32_e32 v2, s2
; GFX6-NEXT: ds_write_b8 v0, v1 offset:9
; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: s_lshr_b32 s2, s2, 16
@@ -224,6 +223,7 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX6-NEXT: s_lshr_b32 s0, s0, 16
; GFX6-NEXT: ds_write_b8 v0, v1 offset:3
; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: ds_write_b8 v0, v2 offset:8
; GFX6-NEXT: ds_write_b8 v0, v1 offset:2
; GFX6-NEXT: s_endpgm
;
@@ -236,35 +236,35 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX10-NEXT: v_mov_b32_e32 v0, s6
; GFX10-NEXT: v_mov_b32_e32 v1, s3
; GFX10-NEXT: v_mov_b32_e32 v2, s2
-; GFX10-NEXT: s_lshr_b32 s5, s2, 8
-; GFX10-NEXT: s_lshr_b32 s2, s2, 24
; GFX10-NEXT: v_mov_b32_e32 v3, s1
-; GFX10-NEXT: s_lshr_b32 s4, s3, 8
-; GFX10-NEXT: s_lshr_b32 s3, s3, 24
; GFX10-NEXT: s_lshr_b32 s6, s1, 8
; GFX10-NEXT: s_lshr_b32 s1, s1, 24
-; GFX10-NEXT: v_mov_b32_e32 v8, s2
; GFX10-NEXT: v_mov_b32_e32 v4, s0
+; GFX10-NEXT: s_lshr_b32 s4, s3, 8
+; GFX10-NEXT: s_lshr_b32 s3, s3, 24
+; GFX10-NEXT: s_lshr_b32 s5, s2, 8
+; GFX10-NEXT: s_lshr_b32 s2, s2, 24
+; GFX10-NEXT: ds_write_b8 v0, v1 offset:12
+; GFX10-NEXT: ds_write_b8_d16_hi v0, v1 offset:14
+; GFX10-NEXT: v_mov_b32_e32 v1, s1
+; GFX10-NEXT: s_lshr_b32 s1, s0, 8
+; GFX10-NEXT: s_lshr_b32 s0, s0, 24
; GFX10-NEXT: v_mov_b32_e32 v5, s4
; GFX10-NEXT: v_mov_b32_e32 v6, s3
; GFX10-NEXT: v_mov_b32_e32 v7, s5
+; GFX10-NEXT: v_mov_b32_e32 v8, s2
; GFX10-NEXT: v_mov_b32_e32 v9, s6
-; GFX10-NEXT: ds_write_b8 v0, v1 offset:12
-; GFX10-NEXT: ds_write_b8_d16_hi v0, v1 offset:14
; GFX10-NEXT: ds_write_b8 v0, v2 offset:8
; GFX10-NEXT: ds_write_b8_d16_hi v0, v2 offset:10
; GFX10-NEXT: ds_write_b8 v0, v3 offset:4
; GFX10-NEXT: ds_write_b8_d16_hi v0, v3 offset:6
+; GFX10-NEXT: v_mov_b32_e32 v2, s1
+; GFX10-NEXT: v_mov_b32_e32 v3, s0
; GFX10-NEXT: ds_write_b8 v0, v4
; GFX10-NEXT: ds_write_b8_d16_hi v0, v4 offset:2
; GFX10-NEXT: ds_write_b8 v0, v5 offset:13
; GFX10-NEXT: ds_write_b8 v0, v6 offset:15
; GFX10-NEXT: ds_write_b8 v0, v7 offset:9
-; GFX10-NEXT: v_mov_b32_e32 v1, s1
-; GFX10-NEXT: s_lshr_b32 s1, s0, 8
-; GFX10-NEXT: s_lshr_b32 s0, s0, 24
-; GFX10-NEXT: v_mov_b32_e32 v2, s1
-; GFX10-NEXT: v_mov_b32_e32 v3, s0
; GFX10-NEXT: ds_write_b8 v0, v8 offset:11
; GFX10-NEXT: ds_write_b8 v0, v9 offset:5
; GFX10-NEXT: ds_write_b8 v0, v1 offset:7
@@ -280,36 +280,36 @@ define amdgpu_kernel void @store_lds_v4i32_align1(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s3
; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s1
+; GFX11-NEXT: s_lshr_b32 s6, s1, 8
+; GFX11-NEXT: s_lshr_b32 s1, s1, 24
; GFX11-NEXT: s_lshr_b32 s4, s3, 8
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v4, s0 :: v_dual_mov_b32 v5, s4
+; GFX11-NEXT: s_lshr_b32 s3, s3, 24
; GFX11-NEXT: s_lshr_b32 s5, s2, 8
; GFX11-NEXT: s_lshr_b32 s2, s2, 24
-; GFX11-NEXT: s_lshr_b32 s6, s1, 8
-; GFX11-NEXT: s_lshr_b32 s3, s3, 24
-; GFX11-NEXT: s_lshr_b32 s1, s1, 24
-; GFX11-NEXT: s_lshr_b32 s7, s0, 8
+; GFX11-NEXT: ds_store_b8 v0, v1 offset:12
+; GFX11-NEXT: ds_store_b8_d16_hi v0, v1 offset:14
+; GFX11-NEXT: v_mov_b32_e32 v1, s1
+; GFX11-NEXT: s_lshr_b32 s1, s0, 8
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
-; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s6
; GFX11-NEXT: v_dual_mov_b32 v6, s3 :: v_dual_mov_b32 v7, s5
-; GFX11-NEXT: v_mov_b32_e32 v10, s1
+; GFX11-NEXT: v_dual_mov_b32 v8, s2 :: v_dual_mov_b32 v9, s6
; GFX11-NEXT: ds_store_b8 v0, v2 offset:8
; GFX11-NEXT: ds_store_b8_d16_hi v0, v2 offset:10
-; GFX11-NEXT: ds_store_b8 v0, v1 offset:12
-; GFX11-NEXT: ds_store_b8_d16_hi v0, v1 offset:14
+; GFX11-NEXT: ds_store_b8 v0, v3 offset:4
+; GFX11-NEXT: ds_store_b8_d16_hi v0, v3 offset:6
+; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s0
+; GFX11-NEXT: ds_store_b8 v0, v4
+; GFX11-NEXT: ds_store_b8_d16_hi v0, v4 offset:2
; GFX11-NEXT: ds_store_b8 v0, v5 offset:13
; GFX11-NEXT: ds_store_b8 v0, v6 offset:15
; GFX11-NEXT: ds_store_b8 v0, v7 offset:9
; GFX11-NEXT: ds_store_b8 v0, v8 offset:11
-; GFX11-NEXT: v_dual_mov_b32 v1, s7 :: v_dual_mov_b32 v2, s0
-; GFX11-NEXT: ds_store_b8 v0, v4
-; GFX11-NEXT: ds_store_b8_d16_hi v0, v4 offset:2
-; GFX11-NEXT: ds_store_b8 v0, v3 offset:4
-; GFX11-NEXT: ds_store_b8_d16_hi v0, v3 offset:6
; GFX11-NEXT: ds_store_b8 v0, v9 offset:5
-; GFX11-NEXT: ds_store_b8 v0, v10 offset:7
-; GFX11-NEXT: ds_store_b8 v0, v1 offset:1
-; GFX11-NEXT: ds_store_b8 v0, v2 offset:3
+; GFX11-NEXT: ds_store_b8 v0, v1 offset:7
+; GFX11-NEXT: ds_store_b8 v0, v2 offset:1
+; GFX11-NEXT: ds_store_b8 v0, v3 offset:3
; GFX11-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out, align 1
ret void
@@ -323,15 +323,15 @@ define amdgpu_kernel void @store_lds_v4i32_align2(ptr addrspace(3) %out, <4 x i3
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s6
; GFX9-NEXT: v_mov_b32_e32 v1, s3
-; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: ds_write_b16 v0, v1 offset:12
; GFX9-NEXT: ds_write_b16_d16_hi v0, v1 offset:14
-; GFX9-NEXT: ds_write_b16 v0, v2 offset:8
-; GFX9-NEXT: ds_write_b16_d16_hi v0, v2 offset:10
; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: v_mov_b32_e32 v2, s2
; GFX9-NEXT: ds_write_b16 v0, v1 offset:4
; GFX9-NEXT: ds_write_b16_d16_hi v0, v1 offset:6
; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: ds_write_b16 v0, v2 offset:8
+; GFX9-NEXT: ds_write_b16_d16_hi v0, v2 offset:10
; GFX9-NEXT: ds_write_b16 v0, v1
; GFX9-NEXT: ds_write_b16_d16_hi v0, v1 offset:2
; GFX9-NEXT: s_endpgm
@@ -344,13 +344,12 @@ define amdgpu_kernel void @store_lds_v4i32_align2(ptr addrspace(3) %out, <4 x i3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, s6
; GFX7-NEXT: v_mov_b32_e32 v1, s3
-; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: ds_write_b16 v0, v1 offset:12
-; GFX7-NEXT: ds_write_b16 v0, v2 offset:8
; GFX7-NEXT: v_mov_b32_e32 v1, s1
; GFX7-NEXT: ds_write_b16 v0, v1 offset:4
; GFX7-NEXT: v_mov_b32_e32 v1, s0
; GFX7-NEXT: s_lshr_b32 s3, s3, 16
+; GFX7-NEXT: v_mov_b32_e32 v2, s2
; GFX7-NEXT: ds_write_b16 v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s3
; GFX7-NEXT: s_lshr_b32 s2, s2, 16
@@ -362,6 +361,7 @@ define amdgpu_kernel void @store_lds_v4i32_align2(ptr addrspace(3) %out, <4 x i3
; GFX7-NEXT: s_lshr_b32 s0, s0, 16
; GFX7-NEXT: ds_write_b16 v0, v1 offset:6
; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: ds_write_b16 v0, v2 offset:8
; GFX7-NEXT: ds_write_b16 v0, v1 offset:2
; GFX7-NEXT: s_endpgm
;
@@ -373,13 +373,12 @@ define amdgpu_kernel void @store_lds_v4i32_align2(ptr addrspace(3) %out, <4 x i3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, s6
; GFX6-NEXT: v_mov_b32_e32 v1, s3
-; GFX6-NEXT: v_mov_b32_e32 v2, s2
; GFX6-NEXT: ds_write_b16 v0, v1 offset:12
-; GFX6-NEXT: ds_write_b16 v0, v2 offset:8
; GFX6-NEXT: v_mov_b32_e32 v1, s1
; GFX6-NEXT: ds_write_b16 v0, v1 offset:4
; GFX6-NEXT: v_mov_b32_e32 v1, s0
; GFX6-NEXT: s_lshr_b32 s3, s3, 16
+; GFX6-NEXT: v_mov_b32_e32 v2, s2
; GFX6-NEXT: ds_write_b16 v0, v1
; GFX6-NEXT: v_mov_b32_e32 v1, s3
; GFX6-NEXT: s_lshr_b32 s2, s2, 16
@@ -391,6 +390,7 @@ define amdgpu_kernel void @store_lds_v4i32_align2(ptr addrspace(3) %out, <4 x i3
; GFX6-NEXT: s_lshr_b32 s0, s0, 16
; GFX6-NEXT: ds_write_b16 v0, v1 offset:6
; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: ds_write_b16 v0, v2 offset:8
; GFX6-NEXT: ds_write_b16 v0, v1 offset:2
; GFX6-NEXT: s_endpgm
;
@@ -421,17 +421,17 @@ define amdgpu_kernel void @store_lds_v4i32_align2(ptr addrspace(3) %out, <4 x i3
; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x0
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s0
-; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
-; GFX11-NEXT: v_mov_b32_e32 v4, s3
-; GFX11-NEXT: ds_store_b16 v0, v1
-; GFX11-NEXT: ds_store_b16 v0, v2 offset:4
-; GFX11-NEXT: ds_store_b16 v0, v3 offset:8
-; GFX11-NEXT: ds_store_b16 v0, v4 offset:12
-; GFX11-NEXT: ds_store_b16_d16_hi v0, v4 offset:14
-; GFX11-NEXT: ds_store_b16_d16_hi v0, v3 offset:10
-; GFX11-NEXT: ds_store_b16_d16_hi v0, v2 offset:6
-; GFX11-NEXT: ds_store_b16_d16_hi v0, v1 offset:2
+; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s3
+; GFX11-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s1
+; GFX11-NEXT: v_mov_b32_e32 v4, s0
+; GFX11-NEXT: ds_store_b16 v0, v1 offset:12
+; GFX11-NEXT: ds_store_b16_d16_hi v0, v1 offset:14
+; GFX11-NEXT: ds_store_b16 v0, v2 offset:8
+; GFX11-NEXT: ds_store_b16_d16_hi v0, v2 offset:10
+; GFX11-NEXT: ds_store_b16 v0, v3 offset:4
+; GFX11-NEXT: ds_store_b16_d16_hi v0, v3 offset:6
+; GFX11-NEXT: ds_store_b16 v0, v4
+; GFX11-NEXT: ds_store_b16_d16_hi v0, v4 offset:2
; GFX11-NEXT: s_endpgm
store <4 x i32> %x, ptr addrspace(3) %out, align 2
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/store-local.96.ll b/llvm/test/CodeGen/AMDGPU/store-local.96.ll
index 3034711f06262..6037849d45550 100644
--- a/llvm/test/CodeGen/AMDGPU/store-local.96.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-local.96.ll
@@ -84,11 +84,8 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX9-NEXT: s_waitcnt lgkmcnt(0)
; GFX9-NEXT: v_mov_b32_e32 v0, s6
; GFX9-NEXT: v_mov_b32_e32 v1, s2
-; GFX9-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NEXT: ds_write_b8 v0, v1 offset:8
; GFX9-NEXT: ds_write_b8_d16_hi v0, v1 offset:10
-; GFX9-NEXT: ds_write_b8 v0, v2 offset:4
-; GFX9-NEXT: ds_write_b8_d16_hi v0, v2 offset:6
; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: s_lshr_b32 s3, s2, 8
; GFX9-NEXT: ds_write_b8 v0, v1
@@ -98,6 +95,7 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX9-NEXT: ds_write_b8 v0, v1 offset:9
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: s_lshr_b32 s2, s1, 8
+; GFX9-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NEXT: ds_write_b8 v0, v1 offset:11
; GFX9-NEXT: v_mov_b32_e32 v1, s2
; GFX9-NEXT: s_lshr_b32 s1, s1, 24
@@ -109,6 +107,8 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX9-NEXT: s_lshr_b32 s0, s0, 24
; GFX9-NEXT: ds_write_b8 v0, v1 offset:1
; GFX9-NEXT: v_mov_b32_e32 v1, s0
+; GFX9-NEXT: ds_write_b8 v0, v2 offset:4
+; GFX9-NEXT: ds_write_b8_d16_hi v0, v2 offset:6
; GFX9-NEXT: ds_write_b8 v0, v1 offset:3
; GFX9-NEXT: s_endpgm
;
@@ -120,9 +120,7 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, s6
; GFX7-NEXT: v_mov_b32_e32 v1, s2
-; GFX7-NEXT: v_mov_b32_e32 v2, s1
; GFX7-NEXT: ds_write_b8 v0, v1 offset:8
-; GFX7-NEXT: ds_write_b8 v0, v2 offset:4
; GFX7-NEXT: v_mov_b32_e32 v1, s0
; GFX7-NEXT: s_lshr_b32 s3, s2, 8
; GFX7-NEXT: ds_write_b8 v0, v1
@@ -137,6 +135,7 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX7-NEXT: ds_write_b8 v0, v1 offset:10
; GFX7-NEXT: v_mov_b32_e32 v1, s2
; GFX7-NEXT: s_lshr_b32 s2, s1, 24
+; GFX7-NEXT: v_mov_b32_e32 v2, s1
; GFX7-NEXT: ds_write_b8 v0, v1 offset:5
; GFX7-NEXT: v_mov_b32_e32 v1, s2
; GFX7-NEXT: s_lshr_b32 s1, s1, 16
@@ -151,6 +150,7 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX7-NEXT: s_lshr_b32 s0, s0, 16
; GFX7-NEXT: ds_write_b8 v0, v1 offset:3
; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: ds_write_b8 v0, v2 offset:4
; GFX7-NEXT: ds_write_b8 v0, v1 offset:2
; GFX7-NEXT: s_endpgm
;
@@ -162,9 +162,7 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, s6
; GFX6-NEXT: v_mov_b32_e32 v1, s2
-; GFX6-NEXT: v_mov_b32_e32 v2, s1
; GFX6-NEXT: ds_write_b8 v0, v1 offset:8
-; GFX6-NEXT: ds_write_b8 v0, v2 offset:4
; GFX6-NEXT: v_mov_b32_e32 v1, s0
; GFX6-NEXT: s_lshr_b32 s3, s2, 8
; GFX6-NEXT: ds_write_b8 v0, v1
@@ -179,6 +177,7 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX6-NEXT: ds_write_b8 v0, v1 offset:10
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: s_lshr_b32 s2, s1, 24
+; GFX6-NEXT: v_mov_b32_e32 v2, s1
; GFX6-NEXT: ds_write_b8 v0, v1 offset:5
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: s_lshr_b32 s1, s1, 16
@@ -193,6 +192,7 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX6-NEXT: s_lshr_b32 s0, s0, 16
; GFX6-NEXT: ds_write_b8 v0, v1 offset:3
; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: ds_write_b8 v0, v2 offset:4
; GFX6-NEXT: ds_write_b8 v0, v1 offset:2
; GFX6-NEXT: s_endpgm
;
@@ -239,25 +239,24 @@ define amdgpu_kernel void @store_lds_v3i32_align1(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s0
; GFX11-NEXT: s_lshr_b32 s3, s2, 8
; GFX11-NEXT: s_lshr_b32 s2, s2, 24
-; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s2
-; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s0
; GFX11-NEXT: s_lshr_b32 s4, s1, 8
; GFX11-NEXT: s_lshr_b32 s1, s1, 24
; GFX11-NEXT: s_lshr_b32 s5, s0, 8
; GFX11-NEXT: s_lshr_b32 s0, s0, 24
+; GFX11-NEXT: v_dual_mov_b32 v4, s3 :: v_dual_mov_b32 v5, s2
; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v7, s1
; GFX11-NEXT: v_dual_mov_b32 v8, s5 :: v_dual_mov_b32 v9, s0
; GFX11-NEXT: ds_store_b8 v0, v1 offset:8
; GFX11-NEXT: ds_store_b8_d16_hi v0, v1 offset:10
-; GFX11-NEXT: ds_store_b8 v0, v4 offset:9
-; GFX11-NEXT: ds_store_b8 v0, v5 offset:11
-; GFX11-NEXT: ds_store_b8 v0, v3
-; GFX11-NEXT: ds_store_b8_d16_hi v0, v3 offset:2
; GFX11-NEXT: ds_store_b8 v0, v2 offset:4
; GFX11-NEXT: ds_store_b8_d16_hi v0, v2 offset:6
+; GFX11-NEXT: ds_store_b8 v0, v3
+; GFX11-NEXT: ds_store_b8_d16_hi v0, v3 offset:2
+; GFX11-NEXT: ds_store_b8 v0, v4 offset:9
+; GFX11-NEXT: ds_store_b8 v0, v5 offset:11
; GFX11-NEXT: ds_store_b8 v0, v6 offset:5
; GFX11-NEXT: ds_store_b8 v0, v7 offset:7
; GFX11-NEXT: ds_store_b8 v0, v8 offset:1
@@ -278,9 +277,9 @@ define amdgpu_kernel void @store_lds_v3i32_align2(ptr addrspace(3) %out, <3 x i3
; GFX9-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NEXT: ds_write_b16 v0, v1 offset:8
; GFX9-NEXT: ds_write_b16_d16_hi v0, v1 offset:10
+; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: ds_write_b16 v0, v2 offset:4
; GFX9-NEXT: ds_write_b16_d16_hi v0, v2 offset:6
-; GFX9-NEXT: v_mov_b32_e32 v1, s0
; GFX9-NEXT: ds_write_b16 v0, v1
; GFX9-NEXT: ds_write_b16_d16_hi v0, v1 offset:2
; GFX9-NEXT: s_endpgm
@@ -293,11 +292,10 @@ define amdgpu_kernel void @store_lds_v3i32_align2(ptr addrspace(3) %out, <3 x i3
; GFX7-NEXT: s_waitcnt lgkmcnt(0)
; GFX7-NEXT: v_mov_b32_e32 v0, s6
; GFX7-NEXT: v_mov_b32_e32 v1, s2
-; GFX7-NEXT: v_mov_b32_e32 v2, s1
; GFX7-NEXT: ds_write_b16 v0, v1 offset:8
-; GFX7-NEXT: ds_write_b16 v0, v2 offset:4
; GFX7-NEXT: v_mov_b32_e32 v1, s0
; GFX7-NEXT: s_lshr_b32 s2, s2, 16
+; GFX7-NEXT: v_mov_b32_e32 v2, s1
; GFX7-NEXT: ds_write_b16 v0, v1
; GFX7-NEXT: v_mov_b32_e32 v1, s2
; GFX7-NEXT: s_lshr_b32 s1, s1, 16
@@ -306,6 +304,7 @@ define amdgpu_kernel void @store_lds_v3i32_align2(ptr addrspace(3) %out, <3 x i3
; GFX7-NEXT: s_lshr_b32 s0, s0, 16
; GFX7-NEXT: ds_write_b16 v0, v1 offset:6
; GFX7-NEXT: v_mov_b32_e32 v1, s0
+; GFX7-NEXT: ds_write_b16 v0, v2 offset:4
; GFX7-NEXT: ds_write_b16 v0, v1 offset:2
; GFX7-NEXT: s_endpgm
;
@@ -317,11 +316,10 @@ define amdgpu_kernel void @store_lds_v3i32_align2(ptr addrspace(3) %out, <3 x i3
; GFX6-NEXT: s_waitcnt lgkmcnt(0)
; GFX6-NEXT: v_mov_b32_e32 v0, s6
; GFX6-NEXT: v_mov_b32_e32 v1, s2
-; GFX6-NEXT: v_mov_b32_e32 v2, s1
; GFX6-NEXT: ds_write_b16 v0, v1 offset:8
-; GFX6-NEXT: ds_write_b16 v0, v2 offset:4
; GFX6-NEXT: v_mov_b32_e32 v1, s0
; GFX6-NEXT: s_lshr_b32 s2, s2, 16
+; GFX6-NEXT: v_mov_b32_e32 v2, s1
; GFX6-NEXT: ds_write_b16 v0, v1
; GFX6-NEXT: v_mov_b32_e32 v1, s2
; GFX6-NEXT: s_lshr_b32 s1, s1, 16
@@ -330,6 +328,7 @@ define amdgpu_kernel void @store_lds_v3i32_align2(ptr addrspace(3) %out, <3 x i3
; GFX6-NEXT: s_lshr_b32 s0, s0, 16
; GFX6-NEXT: ds_write_b16 v0, v1 offset:6
; GFX6-NEXT: v_mov_b32_e32 v1, s0
+; GFX6-NEXT: ds_write_b16 v0, v2 offset:4
; GFX6-NEXT: ds_write_b16 v0, v1 offset:2
; GFX6-NEXT: s_endpgm
;
@@ -357,14 +356,14 @@ define amdgpu_kernel void @store_lds_v3i32_align2(ptr addrspace(3) %out, <3 x i3
; GFX11-NEXT: s_load_b32 s6, s[4:5], 0x0
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x10
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s0
-; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s2
-; GFX11-NEXT: ds_store_b16 v0, v1
+; GFX11-NEXT: v_dual_mov_b32 v0, s6 :: v_dual_mov_b32 v1, s2
+; GFX11-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s0
+; GFX11-NEXT: ds_store_b16 v0, v1 offset:8
+; GFX11-NEXT: ds_store_b16_d16_hi v0, v1 offset:10
; GFX11-NEXT: ds_store_b16 v0, v2 offset:4
-; GFX11-NEXT: ds_store_b16 v0, v3 offset:8
-; GFX11-NEXT: ds_store_b16_d16_hi v0, v3 offset:10
; GFX11-NEXT: ds_store_b16_d16_hi v0, v2 offset:6
-; GFX11-NEXT: ds_store_b16_d16_hi v0, v1 offset:2
+; GFX11-NEXT: ds_store_b16 v0, v3
+; GFX11-NEXT: ds_store_b16_d16_hi v0, v3 offset:2
; GFX11-NEXT: s_endpgm
store <3 x i32> %x, ptr addrspace(3) %out, align 2
ret void
diff --git a/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll b/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
index 480eb0dd5fe9c..623141d50ae98 100644
--- a/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
@@ -11,9 +11,9 @@ define void @local_store_i56(ptr addrspace(3) %ptr, i56 %arg) #0 {
; CIVI: ; %bb.0:
; CIVI-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; CIVI-NEXT: s_mov_b32 m0, -1
-; CIVI-NEXT: ds_write_b16 v0, v2 offset:4
; CIVI-NEXT: ds_write_b32 v0, v1
; CIVI-NEXT: v_lshrrev_b32_e32 v1, 16, v2
+; CIVI-NEXT: ds_write_b16 v0, v2 offset:4
; CIVI-NEXT: ds_write_b8 v0, v1 offset:6
; CIVI-NEXT: s_waitcnt lgkmcnt(0)
; CIVI-NEXT: s_setpc_b64 s[30:31]
@@ -67,10 +67,10 @@ define amdgpu_kernel void @local_store_i55(ptr addrspace(3) %ptr, i55 %arg) #0 {
; HAWAII-NEXT: v_mov_b32_e32 v2, s1
; HAWAII-NEXT: v_mov_b32_e32 v3, s0
; HAWAII-NEXT: ds_write_b16 v1, v2 offset:4
+; HAWAII-NEXT: ds_write_b32 v1, v3
; HAWAII-NEXT: s_waitcnt vmcnt(0)
; HAWAII-NEXT: v_and_b32_e32 v0, 0x7f, v0
; HAWAII-NEXT: ds_write_b8 v1, v0 offset:6
-; HAWAII-NEXT: ds_write_b32 v1, v3
; HAWAII-NEXT: s_endpgm
;
; FIJI-LABEL: local_store_i55:
@@ -93,12 +93,12 @@ define amdgpu_kernel void @local_store_i55(ptr addrspace(3) %ptr, i55 %arg) #0 {
; FIJI-NEXT: s_waitcnt lgkmcnt(0)
; FIJI-NEXT: v_mov_b32_e32 v1, s2
; FIJI-NEXT: ds_write_b16 v1, v2 offset:4
+; FIJI-NEXT: ds_write_b32 v1, v3
; FIJI-NEXT: s_waitcnt vmcnt(0)
; FIJI-NEXT: v_lshlrev_b32_e32 v0, 16, v0
; FIJI-NEXT: v_or_b32_e32 v0, s4, v0
; FIJI-NEXT: v_bfe_u32 v0, v0, 16, 7
; FIJI-NEXT: ds_write_b8 v1, v0 offset:6
-; FIJI-NEXT: ds_write_b32 v1, v3
; FIJI-NEXT: s_endpgm
;
; GFX9-LABEL: local_store_i55:
@@ -113,11 +113,11 @@ define amdgpu_kernel void @local_store_i55(ptr addrspace(3) %ptr, i55 %arg) #0 {
; GFX9-NEXT: v_mov_b32_e32 v2, s1
; GFX9-NEXT: v_mov_b32_e32 v3, s0
; GFX9-NEXT: ds_write_b16 v1, v2 offset:4
+; GFX9-NEXT: ds_write_b32 v1, v3
; GFX9-NEXT: s_waitcnt vmcnt(0)
; GFX9-NEXT: v_or_b32_e32 v0, s3, v0
; GFX9-NEXT: v_and_b32_e32 v0, 0x7fffff, v0
; GFX9-NEXT: ds_write_b8_d16_hi v1, v0 offset:6
-; GFX9-NEXT: ds_write_b32 v1, v3
; GFX9-NEXT: s_endpgm
;
; GFX10-LABEL: local_store_i55:
@@ -132,12 +132,12 @@ define amdgpu_kernel void @local_store_i55(ptr addrspace(3) %ptr, i55 %arg) #0 {
; GFX10-NEXT: v_mov_b32_e32 v1, s2
; GFX10-NEXT: v_mov_b32_e32 v2, s1
; GFX10-NEXT: v_mov_b32_e32 v3, s0
+; GFX10-NEXT: ds_write_b16 v1, v2 offset:4
+; GFX10-NEXT: ds_write_b32 v1, v3
; GFX10-NEXT: s_waitcnt vmcnt(0)
; GFX10-NEXT: v_or_b32_e32 v0, s3, v0
; GFX10-NEXT: v_and_b32_e32 v0, 0x7fffff, v0
-; GFX10-NEXT: ds_write_b16 v1, v2 offset:4
; GFX10-NEXT: ds_write_b8_d16_hi v1, v0 offset:6
-; GFX10-NEXT: ds_write_b32 v1, v3
; GFX10-NEXT: s_endpgm
;
; GFX11-TRUE16-LABEL: local_store_i55:
@@ -155,9 +155,9 @@ define amdgpu_kernel void @local_store_i55(ptr addrspace(3) %ptr, i55 %arg) #0 {
; GFX11-TRUE16-NEXT: v_or_b32_e32 v0, s3, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_and_b32 v0, 0x7fffff, v0
-; GFX11-TRUE16-NEXT: ds_store_b8_d16_hi v1, v0 offset:6
; GFX11-TRUE16-NEXT: ds_store_b16 v1, v2 offset:4
; GFX11-TRUE16-NEXT: ds_store_b32 v1, v3
+; GFX11-TRUE16-NEXT: ds_store_b8_d16_hi v1, v0 offset:6
; GFX11-TRUE16-NEXT: s_endpgm
;
; GFX11-FAKE16-LABEL: local_store_i55:
@@ -171,13 +171,13 @@ define amdgpu_kernel void @local_store_i55(ptr addrspace(3) %ptr, i55 %arg) #0 {
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s2
; GFX11-FAKE16-NEXT: global_load_d16_hi_u8 v0, v0, s[4:5] offset:14
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s0
+; GFX11-FAKE16-NEXT: ds_store_b16 v1, v2 offset:4
+; GFX11-FAKE16-NEXT: ds_store_b32 v1, v3
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, s3, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fffff, v0
; GFX11-FAKE16-NEXT: ds_store_b8_d16_hi v1, v0 offset:6
-; GFX11-FAKE16-NEXT: ds_store_b16 v1, v2 offset:4
-; GFX11-FAKE16-NEXT: ds_store_b32 v1, v3
; GFX11-FAKE16-NEXT: s_endpgm
store i55 %arg, ptr addrspace(3) %ptr, align 8
ret void
@@ -292,8 +292,8 @@ define amdgpu_kernel void @local_store_i65(ptr addrspace(3) %ptr, i65 %arg) #0 {
; GFX9-NEXT: s_and_b32 s2, s2, 1
; GFX9-NEXT: v_mov_b32_e32 v2, s3
; GFX9-NEXT: v_mov_b32_e32 v0, s0
-; GFX9-NEXT: v_mov_b32_e32 v3, s2
; GFX9-NEXT: v_mov_b32_e32 v1, s1
+; GFX9-NEXT: v_mov_b32_e32 v3, s2
; GFX9-NEXT: ds_write_b8 v2, v3 offset:8
; GFX9-NEXT: ds_write_b64 v2, v[0:1]
; GFX9-NEXT: s_endpgm
@@ -307,8 +307,8 @@ define amdgpu_kernel void @local_store_i65(ptr addrspace(3) %ptr, i65 %arg) #0 {
; GFX10-NEXT: s_waitcnt lgkmcnt(0)
; GFX10-NEXT: s_and_b32 s2, s2, 1
; GFX10-NEXT: v_mov_b32_e32 v2, s3
-; GFX10-NEXT: v_mov_b32_e32 v3, s2
; GFX10-NEXT: v_mov_b32_e32 v0, s0
+; GFX10-NEXT: v_mov_b32_e32 v3, s2
; GFX10-NEXT: v_mov_b32_e32 v1, s1
; GFX10-NEXT: ds_write_b8 v2, v3 offset:8
; GFX10-NEXT: ds_write_b64 v2, v[0:1]
More information about the llvm-commits
mailing list