[llvm] [AMDGPU][VOPD] Limit VOPDPairing from reaching over load dependencies (PR #201930)
Zach Goldthorpe via llvm-commits
llvm-commits at lists.llvm.org
Mon Jun 8 14:49:50 PDT 2026
https://github.com/zGoldthorpe updated https://github.com/llvm/llvm-project/pull/201930
>From c1a5b642fbd8cfdaca3477ae3b3529932dcf59bd Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Fri, 5 Jun 2026 14:16:59 -0500
Subject: [PATCH 1/7] Skip VOPD pairs reaching over a load dependency.
---
llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp | 20 ++++++++++++++++++++
1 file changed, 20 insertions(+)
diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
index d6cf6fb37ee47..a89805bb9595c 100644
--- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
@@ -264,6 +264,24 @@ static bool shouldScheduleVOPDAdjacent(const TargetInstrInfo &TII,
.has_value();
}
+/// Check whether \p SU has a load dependency that is currently scheduled after
+/// \p Base (according to node number).
+static bool hasIntermediateLoadDependency(const SUnit &SU, const SUnit &Base) {
+ for (const SDep &Dep : SU.Preds) {
+ if (Dep.getKind() != SDep::Data)
+ continue;
+ const SUnit *Pred = Dep.getSUnit();
+ if (Pred->NodeNum < Base.NodeNum)
+ continue;
+ if (!Pred->isInstr())
+ continue;
+ const MachineInstr *MI = Pred->getInstr();
+ if (MI->mayLoad())
+ return true;
+ }
+ return false;
+}
+
namespace {
/// Adapts design from MacroFusion
/// Puts valid candidate instructions back-to-back so they can easily
@@ -295,6 +313,8 @@ struct VOPDPairingMutation : ScheduleDAGMutation {
for (JSUI = ISUI + 1; JSUI != DAG->SUnits.end(); ++JSUI) {
if (JSUI->isBoundaryNode())
continue;
+ if (hasIntermediateLoadDependency(*JSUI, *ISUI))
+ continue;
const MachineInstr *JMI = JSUI->getInstr();
if (!hasLessThanNumFused(*JSUI, 2) ||
!shouldScheduleAdjacent(TII, ST, IMI, *JMI))
>From dae10167d10eebe703881c0b1fc53fe5e0e8fd97 Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Fri, 5 Jun 2026 14:39:04 -0500
Subject: [PATCH 2/7] Update lit-tests
---
.../AMDGPU/atomicrmw-bf16-gfx11plus.ll | 5 +-
.../AMDGPU/buffer-fat-pointers-memcpy.ll | 4 +-
llvm/test/CodeGen/AMDGPU/ctpop64.ll | 3 +-
.../CodeGen/AMDGPU/dagcombine-fma-fmad.ll | 20 ++--
llvm/test/CodeGen/AMDGPU/fmed3.ll | 10 +-
llvm/test/CodeGen/AMDGPU/fneg.bf16.ll | 16 +--
.../AMDGPU/insert_vector_elt.v2bf16.ll | 11 +-
.../AMDGPU/llvm.amdgcn.av.store.b128.ll | 44 ++++----
.../llvm.amdgcn.struct.buffer.load.format.ll | 33 +++---
...vm.amdgcn.struct.ptr.buffer.load.format.ll | 22 ++--
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 106 +++++++++---------
llvm/test/CodeGen/AMDGPU/load-constant-i8.ll | 8 +-
...er-buffer-fat-pointers-lastuse-metadata.ll | 8 +-
...uffer-fat-pointers-nontemporal-metadata.ll | 13 +--
llvm/test/CodeGen/AMDGPU/madak.ll | 3 +-
llvm/test/CodeGen/AMDGPU/packed-fp32.ll | 56 ++++-----
llvm/test/CodeGen/AMDGPU/scratch-simple.ll | 91 ++++++++-------
llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll | 9 +-
18 files changed, 228 insertions(+), 234 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/atomicrmw-bf16-gfx11plus.ll b/llvm/test/CodeGen/AMDGPU/atomicrmw-bf16-gfx11plus.ll
index e784c33918f9b..42a8dffce6353 100644
--- a/llvm/test/CodeGen/AMDGPU/atomicrmw-bf16-gfx11plus.ll
+++ b/llvm/test/CodeGen/AMDGPU/atomicrmw-bf16-gfx11plus.ll
@@ -77,13 +77,12 @@ define amdgpu_kernel void @v_atomicrmw_fadd_bf16(ptr addrspace(1) %out, i1 %in,
; GFX11-FAKE16-NEXT: s_lshl_b32 s2, s2, 3
; GFX11-FAKE16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-FAKE16-NEXT: s_lshl_b32 s4, 0xffff, s2
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s3
; GFX11-FAKE16-NEXT: s_not_b32 s3, s4
; GFX11-FAKE16-NEXT: s_mov_b32 s4, 0
-; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v0
; GFX11-FAKE16-NEXT: .p2align 6
; GFX11-FAKE16-NEXT: .LBB0_1: ; %atomicrmw.start
diff --git a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
index e2f9160438581..85ac89644a7ab 100644
--- a/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
+++ b/llvm/test/CodeGen/AMDGPU/buffer-fat-pointers-memcpy.ll
@@ -1142,9 +1142,9 @@ define amdgpu_kernel void @memcpy_known_small(ptr addrspace(7) %src, ptr addrspa
; SDAG-GFX1100-NEXT: s_clause 0x1
; SDAG-GFX1100-NEXT: s_load_b128 s[0:3], s[4:5], 0x44
; SDAG-GFX1100-NEXT: s_load_b32 s7, s[4:5], 0x54
+; SDAG-GFX1100-NEXT: buffer_load_b128 v[0:3], v4, s[8:11], 0 offen
; SDAG-GFX1100-NEXT: s_waitcnt lgkmcnt(0)
; SDAG-GFX1100-NEXT: v_mov_b32_e32 v5, s0
-; SDAG-GFX1100-NEXT: buffer_load_b128 v[0:3], v4, s[8:11], 0 offen
; SDAG-GFX1100-NEXT: s_mov_b32 s4, s1
; SDAG-GFX1100-NEXT: s_mov_b32 s5, s2
; SDAG-GFX1100-NEXT: s_mov_b32 s6, s3
@@ -1192,9 +1192,9 @@ define amdgpu_kernel void @memcpy_known_small(ptr addrspace(7) %src, ptr addrspa
; GISEL-GFX1100-NEXT: s_clause 0x1
; GISEL-GFX1100-NEXT: s_load_b128 s[0:3], s[4:5], 0x44
; GISEL-GFX1100-NEXT: s_load_b32 s7, s[4:5], 0x54
+; GISEL-GFX1100-NEXT: buffer_load_b128 v[0:3], v4, s[8:11], 0 offen
; GISEL-GFX1100-NEXT: s_waitcnt lgkmcnt(0)
; GISEL-GFX1100-NEXT: v_mov_b32_e32 v5, s0
-; GISEL-GFX1100-NEXT: buffer_load_b128 v[0:3], v4, s[8:11], 0 offen
; GISEL-GFX1100-NEXT: s_mov_b32 s4, s1
; GISEL-GFX1100-NEXT: s_mov_b32 s5, s2
; GISEL-GFX1100-NEXT: s_mov_b32 s6, s3
diff --git a/llvm/test/CodeGen/AMDGPU/ctpop64.ll b/llvm/test/CodeGen/AMDGPU/ctpop64.ll
index d3d40e4c3c9db..3cf0be07afc6a 100644
--- a/llvm/test/CodeGen/AMDGPU/ctpop64.ll
+++ b/llvm/test/CodeGen/AMDGPU/ctpop64.ll
@@ -163,10 +163,9 @@ define amdgpu_kernel void @v_ctpop_i64_user(ptr addrspace(1) noalias %out, ptr a
;
; GFX12-LABEL: v_ctpop_i64_user:
; GFX12: ; %bb.0:
-; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
; GFX12-NEXT: v_mov_b32_e32 v2, 0
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX12-NEXT: v_lshlrev_b32_e32 v0, 3, v0
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
index 28a18ec3845e0..d27ce714d848b 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
@@ -115,31 +115,29 @@ define amdgpu_ps float @_amdgpu_ps_main() #0 {
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mul_f32_e32 v9, s10, v0
; GFX11-NEXT: v_fma_f32 v0, -v0, s10, s14
-; GFX11-NEXT: v_mul_f32_e32 v3, s22, v3
; GFX11-NEXT: v_dual_fmac_f32 v1, v6, v8 :: v_dual_mul_f32 v8, s18, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_fmac_f32_e32 v9, v0, v6
-; GFX11-NEXT: v_dual_fmac_f32 v10, v7, v6 :: v_dual_mul_f32 v7, v6, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX11-NEXT: v_sub_f32_e32 v0, v1, v5
+; GFX11-NEXT: v_dual_mul_f32 v3, s22, v3 :: v_dual_fmac_f32 v10, v7, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-NEXT: v_dual_fmac_f32 v9, v0, v6 :: v_dual_sub_f32 v0, v1, v5
+; GFX11-NEXT: v_mul_f32_e32 v7, v6, v3
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fma_f32 v3, -v6, v3, v9
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_fmac_f32_e32 v7, v3, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fmac_f32_e32 v5, v0, v6
; GFX11-NEXT: v_mul_f32_e32 v1, v8, v6
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_add_f32_e32 v4, v4, v10
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_dual_mul_f32 v3, v4, v6 :: v_dual_fmaak_f32 v4, s0, v5, 0x3ca3d70a
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fma_f32 v0, v2, s26, -v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_fmac_f32_e32 v1, v0, v6
; GFX11-NEXT: v_mul_f32_e32 v0, v2, v6
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mul_f32_e32 v2, v7, v4
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mul_f32_e32 v1, v3, v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fmac_f32_e32 v1, v2, v0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_max_f32_e32 v0, 0, v1
; GFX11-NEXT: ; return to shader part epilog
.entry:
diff --git a/llvm/test/CodeGen/AMDGPU/fmed3.ll b/llvm/test/CodeGen/AMDGPU/fmed3.ll
index 1909528558ae6..f2e0c653d973a 100644
--- a/llvm/test/CodeGen/AMDGPU/fmed3.ll
+++ b/llvm/test/CodeGen/AMDGPU/fmed3.ll
@@ -5893,10 +5893,9 @@ define amdgpu_kernel void @v_test_safe_med3_f32_pat0_multi_use1(ptr addrspace(1)
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: global_load_b32 v3, v0, s[6:7] glc dlc
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
-; GFX11-SDAG-NEXT: v_max_f32_e32 v1, v1, v1
-; GFX11-SDAG-NEXT: v_dual_max_f32 v3, v3, v3 :: v_dual_max_f32 v2, v2, v2
+; GFX11-SDAG-NEXT: v_dual_max_f32 v1, v1, v1 :: v_dual_max_f32 v2, v2, v2
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-SDAG-NEXT: v_max_f32_e32 v4, v1, v2
+; GFX11-SDAG-NEXT: v_dual_max_f32 v3, v3, v3 :: v_dual_max_f32 v4, v1, v2
; GFX11-SDAG-NEXT: v_min_f32_e32 v3, v4, v3
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_minmax_f32 v1, v1, v2, v3
@@ -8330,13 +8329,12 @@ define amdgpu_kernel void @two_non_inline_constant_multi_use(ptr addrspace(1) %o
; GFX11-GISEL: ; %bb.0:
; GFX11-GISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-GISEL-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, 0x41800000
-; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-GISEL-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-GISEL-NEXT: global_load_b32 v1, v0, s[2:3]
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
-; GFX11-GISEL-NEXT: v_add_f32_e32 v3, 0.5, v1
+; GFX11-GISEL-NEXT: v_dual_mov_b32 v2, 0x41800000 :: v_dual_add_f32 v3, 0.5, v1
; GFX11-GISEL-NEXT: v_med3_f32 v2, v3, 0x41000000, v2
; GFX11-GISEL-NEXT: v_add_f32_e32 v3, 0x41800000, v1
; GFX11-GISEL-NEXT: v_add_f32_e32 v1, 0x41000000, v1
diff --git a/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll b/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
index 63aadaacbeb3a..f2f97faafdd1d 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
@@ -311,13 +311,13 @@ define amdgpu_kernel void @v_fneg_fold_bf16(ptr addrspace(1) %out, ptr addrspace
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc_lo
; GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v0, v1, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
@@ -334,13 +334,13 @@ define amdgpu_kernel void @v_fneg_fold_bf16(ptr addrspace(1) %out, ptr addrspace
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-FAKE16-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, v2, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, v2, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc_lo
; GFX11-FAKE16-NEXT: global_store_d16_hi_b16 v0, v1, s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
index 3d91e82f1b357..37d7c24cb6869 100644
--- a/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/insert_vector_elt.v2bf16.ll
@@ -2169,27 +2169,27 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v9, v3, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 4
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v3
+; GFX1250-FAKE16-NEXT: v_dual_lshrrev_b32 v3, 16, v3 :: v_dual_lshrrev_b32 v10, 16, v2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s3, -1, 0
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 5
-; GFX1250-FAKE16-NEXT: v_dual_lshrrev_b32 v10, 16, v2 :: v_dual_lshrrev_b32 v11, 16, v1
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v2, v2, s4, s3
; GFX1250-FAKE16-NEXT: s_cselect_b32 s3, -1, 0
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 2
+; GFX1250-FAKE16-NEXT: v_dual_lshrrev_b32 v11, 16, v1 :: v_dual_lshrrev_b32 v12, 16, v0
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v3, v3, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 3
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v1, v1, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 0
-; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
-; GFX1250-FAKE16-NEXT: v_dual_lshrrev_b32 v12, 16, v0 :: v_dual_lshrrev_b32 v13, 16, v7
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v11, v11, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 1
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v0, v0, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 14
+; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-FAKE16-NEXT: v_dual_lshrrev_b32 v13, 16, v7 :: v_dual_lshrrev_b32 v14, 16, v6
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v10, v10, s4, s3
; GFX1250-FAKE16-NEXT: v_perm_b32 v3, v3, v9, 0x5040100
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v9, v12, s4, s2
@@ -2198,7 +2198,6 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v7, v7, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 12
-; GFX1250-FAKE16-NEXT: v_dual_lshrrev_b32 v14, 16, v6 :: v_dual_lshrrev_b32 v15, 16, v5
; GFX1250-FAKE16-NEXT: v_perm_b32 v2, v10, v2, 0x5040100
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v10, v13, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
@@ -2206,13 +2205,13 @@ define amdgpu_kernel void @v_insertelement_v16bf16_dynamic(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v6, v6, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 10
+; GFX1250-FAKE16-NEXT: v_dual_lshrrev_b32 v15, 16, v5 :: v_dual_lshrrev_b32 v16, 16, v4
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v12, v14, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 11
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v5, v5, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 8
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v16, 16, v4
; GFX1250-FAKE16-NEXT: v_cndmask_b32_e64 v13, v15, s4, s2
; GFX1250-FAKE16-NEXT: s_cselect_b32 s2, -1, 0
; GFX1250-FAKE16-NEXT: s_cmp_eq_u32 s5, 9
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
index a1de9877cf51a..78c76b698e69b 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.av.store.b128.ll
@@ -1224,15 +1224,15 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs(i32 %voffset,
;
; GFX1100-SDAG-LABEL: global_store_saddr_uniform_ptr_in_vgprs:
; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1100-SDAG-NEXT: s_clause 0x1
; GFX1100-SDAG-NEXT: s_load_b32 s6, s[4:5], 0x24
; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
-; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1100-SDAG-NEXT: ds_load_b64 v[4:5], v0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v6, s6
-; GFX1100-SDAG-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX1100-SDAG-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX1100-SDAG-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
+; GFX1100-SDAG-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
; GFX1100-SDAG-NEXT: v_readfirstlane_b32 s0, v4
; GFX1100-SDAG-NEXT: v_readfirstlane_b32 s1, v5
; GFX1100-SDAG-NEXT: global_store_b128 v6, v[0:3], s[0:1]
@@ -1241,12 +1241,13 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs(i32 %voffset,
; GFX1250-SDAG-LABEL: global_store_saddr_uniform_ptr_in_vgprs:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SDAG-NEXT: s_clause 0x1
; GFX1250-SDAG-NEXT: s_load_b32 s6, s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v6, s6
; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v6, s6
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
@@ -1313,20 +1314,19 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs(i32 %voffset,
;
; GFX1100-ISEL-LABEL: global_store_saddr_uniform_ptr_in_vgprs:
; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1100-ISEL-NEXT: s_clause 0x1
; GFX1100-ISEL-NEXT: s_load_b32 s6, s[4:5], 0x24
; GFX1100-ISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
; GFX1100-ISEL-NEXT: s_mov_b32 s7, 0
-; GFX1100-ISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
; GFX1100-ISEL-NEXT: ds_load_b64 v[0:1], v0
; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-ISEL-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-ISEL-NEXT: v_add_co_u32 v4, vcc_lo, v0, v2
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v3, vcc_lo
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
+; GFX1100-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1100-ISEL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX1100-ISEL-NEXT: global_store_b128 v[4:5], v[0:3], off
; GFX1100-ISEL-NEXT: s_endpgm
;
@@ -1412,15 +1412,15 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs_immoffset(i32
;
; GFX1100-SDAG-LABEL: global_store_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1100-SDAG: ; %bb.0:
+; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1100-SDAG-NEXT: s_clause 0x1
; GFX1100-SDAG-NEXT: s_load_b32 s6, s[4:5], 0x24
; GFX1100-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
-; GFX1100-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1100-SDAG-NEXT: ds_load_b64 v[4:5], v0
; GFX1100-SDAG-NEXT: s_waitcnt lgkmcnt(0)
; GFX1100-SDAG-NEXT: v_mov_b32_e32 v6, s6
-; GFX1100-SDAG-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
-; GFX1100-SDAG-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
+; GFX1100-SDAG-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
+; GFX1100-SDAG-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
; GFX1100-SDAG-NEXT: v_readfirstlane_b32 s0, v4
; GFX1100-SDAG-NEXT: v_readfirstlane_b32 s1, v5
; GFX1100-SDAG-NEXT: global_store_b128 v6, v[0:3], s[0:1] offset:-120
@@ -1429,12 +1429,13 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs_immoffset(i32
; GFX1250-SDAG-LABEL: global_store_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v0, 0
; GFX1250-SDAG-NEXT: s_clause 0x1
; GFX1250-SDAG-NEXT: s_load_b32 s6, s[4:5], 0x24 nv
; GFX1250-SDAG-NEXT: s_load_b128 s[0:3], s[4:5], 0x34 nv
-; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v0, 0 :: v_dual_mov_b32 v6, s6
; GFX1250-SDAG-NEXT: ds_load_b64 v[4:5], v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
+; GFX1250-SDAG-NEXT: v_mov_b32_e32 v6, s6
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[0:1], s[0:1]
; GFX1250-SDAG-NEXT: v_mov_b64_e32 v[2:3], s[2:3]
; GFX1250-SDAG-NEXT: s_wait_dscnt 0x0
@@ -1501,20 +1502,19 @@ define amdgpu_kernel void @global_store_saddr_uniform_ptr_in_vgprs_immoffset(i32
;
; GFX1100-ISEL-LABEL: global_store_saddr_uniform_ptr_in_vgprs_immoffset:
; GFX1100-ISEL: ; %bb.0:
+; GFX1100-ISEL-NEXT: v_mov_b32_e32 v0, 0
; GFX1100-ISEL-NEXT: s_clause 0x1
; GFX1100-ISEL-NEXT: s_load_b32 s6, s[4:5], 0x24
; GFX1100-ISEL-NEXT: s_load_b128 s[0:3], s[4:5], 0x34
; GFX1100-ISEL-NEXT: s_mov_b32 s7, 0
-; GFX1100-ISEL-NEXT: v_mov_b32_e32 v0, 0
-; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
; GFX1100-ISEL-NEXT: ds_load_b64 v[0:1], v0
; GFX1100-ISEL-NEXT: s_waitcnt lgkmcnt(0)
+; GFX1100-ISEL-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
+; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX1100-ISEL-NEXT: v_add_co_u32 v4, vcc_lo, v0, v2
-; GFX1100-ISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX1100-ISEL-NEXT: v_add_co_ci_u32_e64 v5, null, v1, v3, vcc_lo
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v3, s3
-; GFX1100-ISEL-NEXT: v_dual_mov_b32 v1, s1 :: v_dual_mov_b32 v2, s2
+; GFX1100-ISEL-NEXT: v_dual_mov_b32 v0, s0 :: v_dual_mov_b32 v1, s1
+; GFX1100-ISEL-NEXT: v_dual_mov_b32 v2, s2 :: v_dual_mov_b32 v3, s3
; GFX1100-ISEL-NEXT: global_store_b128 v[4:5], v[0:3], off offset:-120
; GFX1100-ISEL-NEXT: s_endpgm
;
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
index 0bbbb0fdb7211..ab65840653c80 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
@@ -153,15 +153,14 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) {
; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092
; GFX11-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092
; GFX11-NEXT: s_mov_b32 s4, 0x8ffc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_f32_e32 v1, v1, v5
; GFX11-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4
-; GFX11-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v1, v5
+; GFX11-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v3, v3, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3
-; GFX11-NEXT: v_add_f32_e32 v2, v10, v2
+; GFX11-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v1, v9, v1
+; GFX11-NEXT: v_dual_add_f32 v2, v10, v2 :: v_dual_add_f32 v3, v11, v3
; GFX11-NEXT: ; return to shader part epilog
;
; NOPRT-LABEL: buffer_load_immoffs_large:
@@ -172,15 +171,14 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) {
; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092
; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092
; NOPRT-NEXT: s_mov_b32 s4, 0x8ffc
-; NOPRT-NEXT: s_waitcnt vmcnt(0)
-; NOPRT-NEXT: v_add_f32_e32 v1, v1, v5
; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4
-; NOPRT-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7
+; NOPRT-NEXT: s_waitcnt vmcnt(1)
+; NOPRT-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v1, v5
+; NOPRT-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v3, v3, v7
; NOPRT-NEXT: s_waitcnt vmcnt(0)
-; NOPRT-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1
; NOPRT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; NOPRT-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3
-; NOPRT-NEXT: v_add_f32_e32 v2, v10, v2
+; NOPRT-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v1, v9, v1
+; NOPRT-NEXT: v_dual_add_f32 v2, v10, v2 :: v_dual_add_f32 v3, v11, v3
; NOPRT-NEXT: ; return to shader part epilog
;
; GFX12-SDAG-LABEL: buffer_load_immoffs_large:
@@ -192,15 +190,14 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(<4 x i32> inreg) {
; GFX12-SDAG-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], s4 idxen offset:4092
; GFX12-SDAG-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s5 idxen offset:4092
; GFX12-SDAG-NEXT: s_mov_b32 s4, 0x8ffc
-; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: v_add_f32_e32 v1, v1, v5
; GFX12-SDAG-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4
-; GFX12-SDAG-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7
+; GFX12-SDAG-NEXT: s_wait_loadcnt 0x1
+; GFX12-SDAG-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v1, v5
+; GFX12-SDAG-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v3, v3, v7
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
-; GFX12-SDAG-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-SDAG-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3
-; GFX12-SDAG-NEXT: v_add_f32_e32 v2, v10, v2
+; GFX12-SDAG-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v1, v9, v1
+; GFX12-SDAG-NEXT: v_dual_add_f32 v2, v10, v2 :: v_dual_add_f32 v3, v11, v3
; GFX12-SDAG-NEXT: ; return to shader part epilog
;
; GFX12-GISEL-LABEL: buffer_load_immoffs_large:
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll
index 63bacf1433e07..584bffcc84dfe 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.ptr.buffer.load.format.ll
@@ -134,15 +134,14 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(ptr addrspace(8) inreg)
; GFX11-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092
; GFX11-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092
; GFX11-NEXT: s_mov_b32 s4, 0x8ffc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_f32_e32 v1, v1, v5
; GFX11-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4
-; GFX11-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7
+; GFX11-NEXT: s_waitcnt vmcnt(1)
+; GFX11-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v1, v5
+; GFX11-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v3, v3, v7
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3
-; GFX11-NEXT: v_add_f32_e32 v2, v10, v2
+; GFX11-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v1, v9, v1
+; GFX11-NEXT: v_dual_add_f32 v2, v10, v2 :: v_dual_add_f32 v3, v11, v3
; GFX11-NEXT: ; return to shader part epilog
;
; NOPRT-LABEL: buffer_load_immoffs_large:
@@ -153,15 +152,14 @@ define amdgpu_ps <4 x float> @buffer_load_immoffs_large(ptr addrspace(8) inreg)
; NOPRT-NEXT: buffer_load_format_xyzw v[0:3], v8, s[0:3], 60 idxen offset:4092
; NOPRT-NEXT: buffer_load_format_xyzw v[4:7], v8, s[0:3], s4 idxen offset:4092
; NOPRT-NEXT: s_mov_b32 s4, 0x8ffc
-; NOPRT-NEXT: s_waitcnt vmcnt(0)
-; NOPRT-NEXT: v_add_f32_e32 v1, v1, v5
; NOPRT-NEXT: buffer_load_format_xyzw v[8:11], v8, s[0:3], s4 idxen offset:4
-; NOPRT-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v3, v3, v7
+; NOPRT-NEXT: s_waitcnt vmcnt(1)
+; NOPRT-NEXT: v_dual_add_f32 v0, v0, v4 :: v_dual_add_f32 v1, v1, v5
+; NOPRT-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v3, v3, v7
; NOPRT-NEXT: s_waitcnt vmcnt(0)
-; NOPRT-NEXT: v_dual_add_f32 v2, v2, v6 :: v_dual_add_f32 v1, v9, v1
; NOPRT-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; NOPRT-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v3, v11, v3
-; NOPRT-NEXT: v_add_f32_e32 v2, v10, v2
+; NOPRT-NEXT: v_dual_add_f32 v0, v8, v0 :: v_dual_add_f32 v1, v9, v1
+; NOPRT-NEXT: v_dual_add_f32 v2, v10, v2 :: v_dual_add_f32 v3, v11, v3
; NOPRT-NEXT: ; return to shader part epilog
main_body:
%d.0 = call <4 x float> @llvm.amdgcn.struct.ptr.buffer.load.format.v4f32(ptr addrspace(8) %0, i32 0, i32 4092, i32 60, i32 0)
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index c22f7132336d4..f2b5c7679dfe9 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -5350,11 +5350,11 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX12-NEXT: global_load_d16_u8 v0, v1, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_and_b32 v0, 1, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_and_b32_e32 v0, 1, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_and_b32 v0, 0xffff, v0
; GFX12-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX12-NEXT: s_endpgm
@@ -5368,12 +5368,12 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: global_load_u8 v0, v1, s[2:3] nv
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_bitop2_b32 v0, 1, v0 bitop3:0x40
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX1250-FAKE16-NEXT: v_dual_lshrrev_b32 v2, 1, v2 :: v_dual_bitop2_b32 v0, 1, v0 bitop3:0x40
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
;
@@ -5586,14 +5586,13 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v0
; GFX12-NEXT: v_bfe_u32 v2, v0, 1, 1
-; GFX12-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_lshrrev_b32_e32 v4, 2, v1
-; GFX12-NEXT: v_mov_b32_e32 v3, v5
-; GFX12-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_and_b32 v2, 0xffff, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_and_b32 v0, 1, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_dual_mov_b32 v3, v5 :: v_dual_and_b32 v4, 0xffff, v4
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
-; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b64 v5, v[4:5], s[0:1] offset:16
; GFX12-NEXT: global_store_b128 v5, v[0:3], s[0:1]
@@ -5764,13 +5763,11 @@ define amdgpu_kernel void @constant_sextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_dual_lshrrev_b32 v2, 2, v0 :: v_dual_lshrrev_b32 v4, 1, v0
; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-NEXT: v_bfe_i32 v6, v2, 0, 1
-; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 31, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX1250-NEXT: v_bfe_i32 v2, v4, 0, 1
-; GFX1250-NEXT: v_ashrrev_i32_e32 v7, 31, v6
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_dual_ashrrev_i32 v1, 31, v0 :: v_dual_ashrrev_i32 v7, 31, v6
; GFX1250-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_b64 v5, v[6:7], s[0:1] offset:16
@@ -5877,10 +5874,10 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: v_lshrrev_b32_e32 v2, 3, v0
; GFX12-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, s3
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10001
+; GFX12-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v3, v1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10001
; GFX12-NEXT: s_and_b32 s2, s2, 1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_b32 s3, 0xffff, s3
@@ -6076,16 +6073,17 @@ define amdgpu_kernel void @constant_sextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-NEXT: global_load_u8 v0, v9, s[2:3] nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_dual_lshrrev_b32 v2, 3, v0 :: v_dual_lshrrev_b32 v4, 2, v0
-; GFX1250-NEXT: v_lshrrev_b32_e32 v8, 1, v0
-; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1250-NEXT: v_bfe_i32 v6, v2, 0, 1
+; GFX1250-NEXT: v_lshrrev_b32_e32 v8, 1, v0
; GFX1250-NEXT: v_bfe_i32 v4, v4, 0, 1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT: v_ashrrev_i32_e32 v7, 31, v6
; GFX1250-NEXT: v_bfe_i32 v2, v8, 0, 1
-; GFX1250-NEXT: v_dual_ashrrev_i32 v1, 31, v0 :: v_dual_ashrrev_i32 v7, 31, v6
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
-; GFX1250-NEXT: v_dual_ashrrev_i32 v5, 31, v4 :: v_dual_ashrrev_i32 v3, 31, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX1250-NEXT: v_ashrrev_i32_e32 v5, 31, v4
+; GFX1250-NEXT: v_dual_ashrrev_i32 v1, 31, v0 :: v_dual_ashrrev_i32 v3, 31, v2
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_b128 v9, v[4:7], s[0:1] offset:16
; GFX1250-NEXT: global_store_b128 v9, v[0:3], s[0:1]
@@ -6225,23 +6223,23 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i64(ptr addrspace(1) %out
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_mov_b32_e32 v7, v1
+; GFX12-NEXT: v_mov_b32_e32 v9, v1
+; GFX12-NEXT: v_mov_b32_e32 v11, v1
+; GFX12-NEXT: v_mov_b32_e32 v13, v1
+; GFX12-NEXT: v_mov_b32_e32 v15, v1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_d16_u8 v12, v1, s[2:3]
+; GFX12-NEXT: v_mov_b32_e32 v3, v1
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_and_b32 v0, 0xffff, v12
-; GFX12-NEXT: v_mov_b32_e32 v5, v1
-; GFX12-NEXT: v_mov_b32_e32 v7, v1
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_and_b32 v0, 0xffff, v12
; GFX12-NEXT: v_bfe_u32 v6, v12, 5, 1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-NEXT: v_lshrrev_b32_e32 v2, 7, v0
-; GFX12-NEXT: v_bfe_u32 v0, v0, 6, 1
; GFX12-NEXT: v_bfe_u32 v4, v12, 4, 1
-; GFX12-NEXT: v_mov_b32_e32 v9, v1
-; GFX12-NEXT: v_mov_b32_e32 v11, v1
; GFX12-NEXT: v_bfe_u32 v10, v12, 3, 1
; GFX12-NEXT: v_bfe_u32 v8, v12, 2, 1
-; GFX12-NEXT: v_mov_b32_e32 v13, v1
-; GFX12-NEXT: v_mov_b32_e32 v15, v1
+; GFX12-NEXT: v_lshrrev_b32_e32 v2, 7, v0
+; GFX12-NEXT: v_bfe_u32 v0, v0, 6, 1
; GFX12-NEXT: v_bfe_u32 v14, v12, 1, 1
; GFX12-NEXT: v_and_b32_e32 v12, 1, v12
; GFX12-NEXT: s_clause 0x3
@@ -6752,21 +6750,22 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12-NEXT: global_load_d16_b16 v0, v3, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-NEXT: v_dual_mov_b32 v7, v3 :: v_dual_and_b32 v6, 0xffff, v0
-; GFX12-NEXT: v_mov_b32_e32 v9, v3
; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000a
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v2, s3
-; GFX12-NEXT: v_bfe_u32 v4, v6, 11, 1
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000c
+; GFX12-NEXT: v_mov_b32_e32 v2, s3
+; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_and_b32 v6, 0xffff, v0
; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10009
; GFX12-NEXT: v_mov_b32_e32 v1, v3
+; GFX12-NEXT: v_mov_b32_e32 v7, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX12-NEXT: v_bfe_u32 v4, v6, 11, 1
; GFX12-NEXT: v_bfe_u32 v0, v6, 8, 1
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000c
+; GFX12-NEXT: v_lshrrev_b32_e32 v8, 15, v6
+; GFX12-NEXT: v_mov_b32_e32 v9, v3
; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v2, s3
; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000d
-; GFX12-NEXT: v_lshrrev_b32_e32 v8, 15, v6
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v4, s3
; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10007
@@ -6810,20 +6809,19 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-FAKE16-NEXT: global_load_u16 v12, v1, s[2:3] nv
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v22, 0xffff, v12
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_bitop2_b32 v28, 1, v12 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, v1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v0, v12, 10, 1
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v2, v22, 11, 1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v9, v1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v6, v12, 9, 1
+; GFX1250-FAKE16-NEXT: v_bfe_u32 v14, v12, 13, 1
+; GFX1250-FAKE16-NEXT: v_bfe_u32 v18, v12, 7, 1
+; GFX1250-FAKE16-NEXT: v_bfe_u32 v2, v22, 11, 1
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_lshrrev_b32 v10, 15, v22
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v7, v1
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v9, v1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v4, v22, 8, 1
; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v13, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v31, v1 :: v_dual_lshrrev_b32 v10, 15, v22
; GFX1250-FAKE16-NEXT: v_bfe_u32 v8, v22, 14, 1
; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v15, v1 :: v_dual_mov_b32 v17, v1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v14, v12, 13, 1
-; GFX1250-FAKE16-NEXT: v_bfe_u32 v18, v12, 7, 1
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v31, v1 :: v_dual_bitop2_b32 v28, 1, v12 bitop3:0x40
; GFX1250-FAKE16-NEXT: v_bfe_u32 v26, v12, 3, 1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v30, v12, 1, 1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v24, v12, 2, 1
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
index 6584a1e2b760e..658d79cccd4ef 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i8.ll
@@ -5638,11 +5638,13 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i64(ptr addrspace(1) %out
; GFX12-TRUE16: ; %bb.0:
; GFX12-TRUE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-TRUE16-NEXT: v_mov_b32_e32 v3, v1
; GFX12-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-TRUE16-NEXT: global_load_d16_b16 v0, v1, s[2:3]
; GFX12-TRUE16-NEXT: s_wait_loadcnt 0x0
; GFX12-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX12-TRUE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_and_b32 v0, 0xff, v0
+; GFX12-TRUE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-TRUE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX12-TRUE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
@@ -5652,11 +5654,13 @@ define amdgpu_kernel void @constant_zextload_v2i8_to_v2i64(ptr addrspace(1) %out
; GFX12-FAKE16: ; %bb.0:
; GFX12-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX12-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-FAKE16-NEXT: global_load_u16 v0, v1, s[2:3]
; GFX12-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX12-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX12-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_and_b32 v0, 0xff, v0
+; GFX12-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 8, v2
; GFX12-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-lastuse-metadata.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-lastuse-metadata.ll
index 316ebb94948f2..e96d2e81a3ee4 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-lastuse-metadata.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-lastuse-metadata.ll
@@ -17,9 +17,9 @@ define amdgpu_kernel void @buffer_last_use_load_0(ptr addrspace(7) %in, ptr addr
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x20
; GFX12-NEXT: s_load_b32 s7, s[4:5], 0x30
+; GFX12-NEXT: buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_LU
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s0
-; GFX12-NEXT: buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_LU
; GFX12-NEXT: s_mov_b32 s4, s1
; GFX12-NEXT: s_mov_b32 s5, s2
; GFX12-NEXT: s_mov_b32 s6, s3
@@ -79,10 +79,10 @@ define amdgpu_kernel void @buffer_last_use_and_volatile_load(ptr addrspace(7) %i
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x20
; GFX12-NEXT: s_load_b32 s7, s[4:5], 0x30
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s0
; GFX12-NEXT: buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_BYPASS scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s0
; GFX12-NEXT: s_mov_b32 s4, s1
; GFX12-NEXT: s_mov_b32 s5, s2
; GFX12-NEXT: s_mov_b32 s6, s3
@@ -108,9 +108,9 @@ define amdgpu_kernel void @buffer_last_use_and_nontemporal_load(ptr addrspace(7)
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x20
; GFX12-NEXT: s_load_b32 s7, s[4:5], 0x30
+; GFX12-NEXT: buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_LU
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s0
-; GFX12-NEXT: buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_LU
; GFX12-NEXT: s_mov_b32 s4, s1
; GFX12-NEXT: s_mov_b32 s5, s2
; GFX12-NEXT: s_mov_b32 s6, s3
diff --git a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll
index 795ab1cfa7c0d..a9db13104e949 100644
--- a/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll
+++ b/llvm/test/CodeGen/AMDGPU/lower-buffer-fat-pointers-nontemporal-metadata.ll
@@ -111,9 +111,9 @@ define amdgpu_kernel void @buffer_nontemporal_load_store(ptr addrspace(7) %in, p
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x20
; GFX11-NEXT: s_load_b32 s7, s[4:5], 0x30
+; GFX11-NEXT: buffer_load_b32 v0, v0, s[8:11], 0 offen slc dlc
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: v_mov_b32_e32 v1, s0
-; GFX11-NEXT: buffer_load_b32 v0, v0, s[8:11], 0 offen slc dlc
; GFX11-NEXT: s_mov_b32 s4, s1
; GFX11-NEXT: s_mov_b32 s5, s2
; GFX11-NEXT: s_mov_b32 s6, s3
@@ -134,9 +134,9 @@ define amdgpu_kernel void @buffer_nontemporal_load_store(ptr addrspace(7) %in, p
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x20
; GFX12-NEXT: s_load_b32 s7, s[4:5], 0x30
+; GFX12-NEXT: buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_NT
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mov_b32_e32 v1, s0
-; GFX12-NEXT: buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_NT
; GFX12-NEXT: s_mov_b32 s4, s1
; GFX12-NEXT: s_mov_b32 s5, s2
; GFX12-NEXT: s_mov_b32 s6, s3
@@ -254,10 +254,9 @@ define amdgpu_kernel void @buffer_nontemporal_and_volatile_load_store(ptr addrsp
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: s_load_b128 s[0:3], s[4:5], 0x20
; GFX11-NEXT: s_load_b32 s7, s[4:5], 0x30
-; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mov_b32_e32 v1, s0
; GFX11-NEXT: buffer_load_b32 v0, v0, s[8:11], 0 offen glc dlc
-; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: s_waitcnt vmcnt(0) lgkmcnt(0)
+; GFX11-NEXT: v_mov_b32_e32 v1, s0
; GFX11-NEXT: s_mov_b32 s4, s1
; GFX11-NEXT: s_mov_b32 s5, s2
; GFX11-NEXT: s_mov_b32 s6, s3
@@ -278,10 +277,10 @@ define amdgpu_kernel void @buffer_nontemporal_and_volatile_load_store(ptr addrsp
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x20
; GFX12-NEXT: s_load_b32 s7, s[4:5], 0x30
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_mov_b32_e32 v1, s0
; GFX12-NEXT: buffer_load_b32 v0, v0, s[8:11], null offen th:TH_LOAD_NT scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_mov_b32_e32 v1, s0
; GFX12-NEXT: s_mov_b32 s4, s1
; GFX12-NEXT: s_mov_b32 s5, s2
; GFX12-NEXT: s_mov_b32 s6, s3
diff --git a/llvm/test/CodeGen/AMDGPU/madak.ll b/llvm/test/CodeGen/AMDGPU/madak.ll
index b16dad4316db4..0855f2c837339 100644
--- a/llvm/test/CodeGen/AMDGPU/madak.ll
+++ b/llvm/test/CodeGen/AMDGPU/madak.ll
@@ -697,10 +697,9 @@ define amdgpu_kernel void @s_v_madak_f32(ptr addrspace(1) noalias %out, ptr addr
;
; GFX11-MAD-LABEL: s_v_madak_f32:
; GFX11-MAD: ; %bb.0:
-; GFX11-MAD-NEXT: s_clause 0x1
; GFX11-MAD-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
-; GFX11-MAD-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-MAD-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX11-MAD-NEXT: s_load_b32 s4, s[4:5], 0x34
; GFX11-MAD-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-MAD-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-MAD-NEXT: s_waitcnt lgkmcnt(0)
diff --git a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
index 27ea6f344796f..cb4808bc7e776 100644
--- a/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
+++ b/llvm/test/CodeGen/AMDGPU/packed-fp32.ll
@@ -348,14 +348,14 @@ define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; GFX1250-SDAG-LABEL: fadd_v32_vs:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: s_clause 0x2
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-SDAG-NEXT: s_clause 0x1
; GFX1250-SDAG-NEXT: s_load_b512 s[36:51], s[4:5], 0xa4 nv
; GFX1250-SDAG-NEXT: s_load_b512 s[8:23], s[4:5], 0xe4 nv
-; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_dual_lshlrev_b32 v56, 7, v0 :: v_dual_mov_b32 v32, s40
+; GFX1250-SDAG-NEXT: v_lshlrev_b32_e32 v56, 7, v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_clause 0x7
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v56, s[0:1] offset:16
; GFX1250-SDAG-NEXT: global_load_b128 v[4:7], v56, s[0:1] offset:48
@@ -365,18 +365,18 @@ define amdgpu_kernel void @fadd_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; GFX1250-SDAG-NEXT: global_load_b128 v[20:23], v56, s[0:1] offset:96
; GFX1250-SDAG-NEXT: global_load_b128 v[24:27], v56, s[0:1] offset:64
; GFX1250-SDAG-NEXT: global_load_b128 v[28:31], v56, s[0:1] offset:112
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v33, s41 :: v_dual_mov_b32 v34, s42
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v35, s43 :: v_dual_mov_b32 v36, s38
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v39, s49 :: v_dual_mov_b32 v40, s50
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v41, s51 :: v_dual_mov_b32 v42, s44
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v37, s39 :: v_dual_mov_b32 v38, s48
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v55, s23 :: v_dual_mov_b32 v51, s11
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v32, s40 :: v_dual_mov_b32 v33, s41
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v34, s42 :: v_dual_mov_b32 v35, s43
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v40, s50 :: v_dual_mov_b32 v41, s51
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v38, s48 :: v_dual_mov_b32 v39, s49
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v52, s20 :: v_dual_mov_b32 v53, s21
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v54, s22 :: v_dual_mov_b32 v49, s15
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v50, s10 :: v_dual_mov_b32 v45, s47
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v54, s22 :: v_dual_mov_b32 v55, s23
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v50, s10 :: v_dual_mov_b32 v51, s11
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v46, s12 :: v_dual_mov_b32 v47, s13
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v48, s14 :: v_dual_mov_b32 v43, s45
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v44, s46
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v48, s14 :: v_dual_mov_b32 v49, s15
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v42, s44 :: v_dual_mov_b32 v43, s45
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v44, s46 :: v_dual_mov_b32 v45, s47
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v36, s38 :: v_dual_mov_b32 v37, s39
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x7
; GFX1250-SDAG-NEXT: v_pk_add_f32 v[0:1], v[0:1], v[32:33]
; GFX1250-SDAG-NEXT: v_pk_add_f32 v[2:3], v[2:3], v[34:35]
@@ -1635,14 +1635,14 @@ define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; GFX1250-SDAG-LABEL: fmul_v32_vs:
; GFX1250-SDAG: ; %bb.0:
; GFX1250-SDAG-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
-; GFX1250-SDAG-NEXT: s_clause 0x2
; GFX1250-SDAG-NEXT: s_load_b64 s[0:1], s[4:5], 0x24 nv
+; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX1250-SDAG-NEXT: s_clause 0x1
; GFX1250-SDAG-NEXT: s_load_b512 s[36:51], s[4:5], 0xa4 nv
; GFX1250-SDAG-NEXT: s_load_b512 s[8:23], s[4:5], 0xe4 nv
-; GFX1250-SDAG-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX1250-SDAG-NEXT: v_dual_lshlrev_b32 v56, 7, v0 :: v_dual_mov_b32 v32, s40
+; GFX1250-SDAG-NEXT: v_lshlrev_b32_e32 v56, 7, v0
+; GFX1250-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX1250-SDAG-NEXT: s_clause 0x7
; GFX1250-SDAG-NEXT: global_load_b128 v[0:3], v56, s[0:1] offset:16
; GFX1250-SDAG-NEXT: global_load_b128 v[4:7], v56, s[0:1] offset:48
@@ -1652,18 +1652,18 @@ define amdgpu_kernel void @fmul_v32_vs(ptr addrspace(1) %a, <32 x float> %x) {
; GFX1250-SDAG-NEXT: global_load_b128 v[20:23], v56, s[0:1] offset:96
; GFX1250-SDAG-NEXT: global_load_b128 v[24:27], v56, s[0:1] offset:64
; GFX1250-SDAG-NEXT: global_load_b128 v[28:31], v56, s[0:1] offset:112
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v33, s41 :: v_dual_mov_b32 v34, s42
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v35, s43 :: v_dual_mov_b32 v36, s38
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v39, s49 :: v_dual_mov_b32 v40, s50
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v41, s51 :: v_dual_mov_b32 v42, s44
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v37, s39 :: v_dual_mov_b32 v38, s48
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v55, s23 :: v_dual_mov_b32 v51, s11
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v32, s40 :: v_dual_mov_b32 v33, s41
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v34, s42 :: v_dual_mov_b32 v35, s43
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v40, s50 :: v_dual_mov_b32 v41, s51
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v38, s48 :: v_dual_mov_b32 v39, s49
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v52, s20 :: v_dual_mov_b32 v53, s21
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v54, s22 :: v_dual_mov_b32 v49, s15
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v50, s10 :: v_dual_mov_b32 v45, s47
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v54, s22 :: v_dual_mov_b32 v55, s23
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v50, s10 :: v_dual_mov_b32 v51, s11
; GFX1250-SDAG-NEXT: v_dual_mov_b32 v46, s12 :: v_dual_mov_b32 v47, s13
-; GFX1250-SDAG-NEXT: v_dual_mov_b32 v48, s14 :: v_dual_mov_b32 v43, s45
-; GFX1250-SDAG-NEXT: v_mov_b32_e32 v44, s46
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v48, s14 :: v_dual_mov_b32 v49, s15
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v42, s44 :: v_dual_mov_b32 v43, s45
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v44, s46 :: v_dual_mov_b32 v45, s47
+; GFX1250-SDAG-NEXT: v_dual_mov_b32 v36, s38 :: v_dual_mov_b32 v37, s39
; GFX1250-SDAG-NEXT: s_wait_loadcnt 0x7
; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[0:1], v[0:1], v[32:33]
; GFX1250-SDAG-NEXT: v_pk_mul_f32 v[2:3], v[2:3], v[34:35]
diff --git a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
index c253f42e0d3c8..5add8082bd81a 100644
--- a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
+++ b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
@@ -977,7 +977,10 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -989,16 +992,14 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
+; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -1981,7 +1982,10 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -1993,16 +1997,14 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
+; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -2985,7 +2987,10 @@ define amdgpu_cs float @cs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -2997,16 +3002,14 @@ define amdgpu_cs float @cs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
+; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -3986,7 +3989,10 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -3998,16 +4004,14 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
+; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -4987,7 +4991,10 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -4999,16 +5006,14 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
+; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -5999,7 +6004,10 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -6011,16 +6019,14 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
+; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -7010,7 +7016,10 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -7022,16 +7031,14 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
+; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
diff --git a/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll b/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
index 480eb0dd5fe9c..eee29e20f96b1 100644
--- a/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
+++ b/llvm/test/CodeGen/AMDGPU/store-weird-sizes.ll
@@ -142,11 +142,11 @@ define amdgpu_kernel void @local_store_i55(ptr addrspace(3) %ptr, i55 %arg) #0 {
;
; GFX11-TRUE16-LABEL: local_store_i55:
; GFX11-TRUE16: ; %bb.0:
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-TRUE16-NEXT: v_mov_b16_e32 v1.l, 0
; GFX11-TRUE16-NEXT: s_clause 0x1
; GFX11-TRUE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX11-TRUE16-NEXT: s_load_b32 s2, s[4:5], 0x0
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-TRUE16-NEXT: global_load_d16_hi_u8 v1, v0, s[4:5] offset:14
; GFX11-TRUE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-TRUE16-NEXT: s_and_b32 s3, s1, 0xffff
@@ -162,19 +162,18 @@ define amdgpu_kernel void @local_store_i55(ptr addrspace(3) %ptr, i55 %arg) #0 {
;
; GFX11-FAKE16-LABEL: local_store_i55:
; GFX11-FAKE16: ; %bb.0:
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-FAKE16-NEXT: s_clause 0x1
; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x8
; GFX11-FAKE16-NEXT: s_load_b32 s2, s[4:5], 0x0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT: global_load_d16_hi_u8 v0, v0, s[4:5] offset:14
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: s_and_b32 s3, s1, 0xffff
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, s2
-; GFX11-FAKE16-NEXT: global_load_d16_hi_u8 v0, v0, s[4:5] offset:14
; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, s1 :: v_dual_mov_b32 v3, s0
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v0, s3, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fffff, v0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_and_b32 v0, 0x7fffff, v0
; GFX11-FAKE16-NEXT: ds_store_b8_d16_hi v1, v0 offset:6
; GFX11-FAKE16-NEXT: ds_store_b16 v1, v2 offset:4
; GFX11-FAKE16-NEXT: ds_store_b32 v1, v3
>From 905dd3cbc94cf663a6d2d8aae80009fa7b7784ae Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Fri, 5 Jun 2026 14:54:01 -0500
Subject: [PATCH 3/7] Add reduced reproducer case
---
.../AMDGPU/consecutive-loads-in-branch.ll | 374 ++++++++++++++++++
1 file changed, 374 insertions(+)
create mode 100644 llvm/test/CodeGen/AMDGPU/consecutive-loads-in-branch.ll
diff --git a/llvm/test/CodeGen/AMDGPU/consecutive-loads-in-branch.ll b/llvm/test/CodeGen/AMDGPU/consecutive-loads-in-branch.ll
new file mode 100644
index 0000000000000..a9d2c876102b0
--- /dev/null
+++ b/llvm/test/CodeGen/AMDGPU/consecutive-loads-in-branch.ll
@@ -0,0 +1,374 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
+; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
+
+define amdgpu_kernel void @straightline(ptr addrspace(1) noalias %inBuf, ptr addrspace(1) noalias %outBuf, i32 %A.base, i32 %B.base, i32 %C.base, i32 %D.base, i32 %stride, i1 %cond) {
+; GFX12-LABEL: straightline:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b256 s[8:15], s[4:5], 0x0
+; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x20
+; GFX12-NEXT: v_and_b32_e32 v1, 0x3ff, v0
+; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_add_nc_u32_e32 v0, s12, v1
+; GFX12-NEXT: v_add_nc_u32_e32 v2, s13, v1
+; GFX12-NEXT: v_add_nc_u32_e32 v4, s14, v1
+; GFX12-NEXT: v_add_nc_u32_e32 v6, s15, v1
+; GFX12-NEXT: s_bitcmp0_b32 s5, 0
+; GFX12-NEXT: s_cbranch_scc1 .LBB0_2
+; GFX12-NEXT: ; %bb.1: ; %fake.pred
+; GFX12-NEXT: s_load_b128 s[0:3], s[8:9], 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, s4, v0
+; GFX12-NEXT: v_add_nc_u32_e32 v2, s4, v2
+; GFX12-NEXT: v_add_nc_u32_e32 v4, s4, v4
+; GFX12-NEXT: v_add_nc_u32_e32 v6, s4, v6
+; GFX12-NEXT: s_branch .LBB0_3
+; GFX12-NEXT: .LBB0_2:
+; GFX12-NEXT: s_mov_b32 s1, 0
+; GFX12-NEXT: s_mov_b32 s2, 0
+; GFX12-NEXT: s_mov_b32 s3, 0
+; GFX12-NEXT: .LBB0_3: ; %main
+; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-NEXT: v_mov_b32_e32 v5, v1
+; GFX12-NEXT: v_lshlrev_b64_e32 v[8:9], 4, v[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v7, v1
+; GFX12-NEXT: v_lshlrev_b64_e32 v[10:11], 4, v[2:3]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT: v_lshlrev_b64_e32 v[12:13], 4, v[4:5]
+; GFX12-NEXT: v_add_co_u32 v8, vcc_lo, s8, v8
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-NEXT: v_lshlrev_b64_e32 v[14:15], 4, v[6:7]
+; GFX12-NEXT: v_add_co_ci_u32_e64 v9, null, s9, v9, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, s8, v10
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, s9, v11, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v10, vcc_lo, s8, v12
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v11, null, s9, v13, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v14, vcc_lo, s8, v14
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v15, null, s9, v15, vcc_lo
+; GFX12-NEXT: s_clause 0x3
+; GFX12-NEXT: global_load_b128 v[2:5], v[8:9], off
+; GFX12-NEXT: global_load_b128 v[6:9], v[6:7], off
+; GFX12-NEXT: global_load_b128 v[10:13], v[10:11], off
+; GFX12-NEXT: global_load_b128 v[14:17], v[14:15], off
+; GFX12-NEXT: s_wait_loadcnt 0x3
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v5, s3, v5
+; GFX12-NEXT: v_add_nc_u32_e32 v4, s2, v4
+; GFX12-NEXT: v_add_nc_u32_e32 v3, s1, v3
+; GFX12-NEXT: v_add_nc_u32_e32 v2, s0, v2
+; GFX12-NEXT: s_wait_loadcnt 0x2
+; GFX12-NEXT: v_add_nc_u32_e32 v9, s3, v9
+; GFX12-NEXT: v_add_nc_u32_e32 v8, s2, v8
+; GFX12-NEXT: v_add_nc_u32_e32 v7, s1, v7
+; GFX12-NEXT: v_add_nc_u32_e32 v6, s0, v6
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_add_nc_u32_e32 v13, s3, v13
+; GFX12-NEXT: v_add_nc_u32_e32 v12, s2, v12
+; GFX12-NEXT: v_add_nc_u32_e32 v11, s1, v11
+; GFX12-NEXT: v_add_nc_u32_e32 v10, s0, v10
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v17, s3, v17
+; GFX12-NEXT: v_add_nc_u32_e32 v16, s2, v16
+; GFX12-NEXT: v_add_nc_u32_e32 v15, s1, v15
+; GFX12-NEXT: v_add_nc_u32_e32 v14, s0, v14
+; GFX12-NEXT: s_clause 0x3
+; GFX12-NEXT: global_store_b128 v1, v[2:5], s[10:11]
+; GFX12-NEXT: global_store_b128 v1, v[6:9], s[10:11] offset:16
+; GFX12-NEXT: global_store_b128 v1, v[10:13], s[10:11] offset:32
+; GFX12-NEXT: global_store_b128 v1, v[14:17], s[10:11] offset:48
+; GFX12-NEXT: s_endpgm
+entry:
+ %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %A.idx.start = add i32 %A.base, %tid
+ %B.idx.start = add i32 %B.base, %tid
+ %C.idx.start = add i32 %C.base, %tid
+ %D.idx.start = add i32 %D.base, %tid
+ br i1 %cond, label %fake.pred, label %main
+
+fake.pred: ; preds = %entry
+ %A.idx.next = add i32 %A.idx.start, %stride
+ %B.idx.next = add i32 %B.idx.start, %stride
+ %C.idx.next = add i32 %C.idx.start, %stride
+ %D.idx.next = add i32 %D.idx.start, %stride
+ %seed.ptr = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 0
+ %seed.acc = load <4 x i32>, ptr addrspace(1) %seed.ptr, align 16
+ br label %main
+
+main: ; preds = %fake.pred, %entry
+ %acc = phi <4 x i32> [ zeroinitializer, %entry ], [ %seed.acc, %fake.pred ]
+ %A.idx = phi i32 [ %A.idx.start, %entry ], [ %A.idx.next, %fake.pred ]
+ %B.idx = phi i32 [ %B.idx.start, %entry ], [ %B.idx.next, %fake.pred ]
+ %C.idx = phi i32 [ %C.idx.start, %entry ], [ %C.idx.next, %fake.pred ]
+ %D.idx = phi i32 [ %D.idx.start, %entry ], [ %D.idx.next, %fake.pred ]
+
+ %A.idx.64 = zext i32 %A.idx to i64
+ %A.ptr = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %A.idx.64
+ %A.data = load <4 x i32>, ptr addrspace(1) %A.ptr, align 16
+ %A.acc = add <4 x i32> %A.data, %acc
+ %B.idx.64 = zext i32 %B.idx to i64
+ %B.ptr = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %B.idx.64
+ %B.data = load <4 x i32>, ptr addrspace(1) %B.ptr, align 16
+ %B.acc = add <4 x i32> %B.data, %acc
+ %C.idx.64 = zext i32 %C.idx to i64
+ %C.ptr = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %C.idx.64
+ %C.data = load <4 x i32>, ptr addrspace(1) %C.ptr, align 16
+ %C.acc = add <4 x i32> %C.data, %acc
+ %D.idx.64 = zext i32 %D.idx to i64
+ %D.ptr = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %D.idx.64
+ %D.data = load <4 x i32>, ptr addrspace(1) %D.ptr, align 16
+ %D.acc = add <4 x i32> %D.data, %acc
+
+ %A.out = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 0
+ %B.out = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 1
+ %C.out = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 2
+ %D.out = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 3
+ store <4 x i32> %A.acc, ptr addrspace(1) %A.out, align 16
+ store <4 x i32> %B.acc, ptr addrspace(1) %B.out, align 16
+ store <4 x i32> %C.acc, ptr addrspace(1) %C.out, align 16
+ store <4 x i32> %D.acc, ptr addrspace(1) %D.out, align 16
+ ret void
+}
+
+define amdgpu_kernel void @loop_with_epilogue(ptr addrspace(1) noalias %inBuf, ptr addrspace(1) noalias %outBuf, i32 %off0, i32 %off1, i32 %off2, i32 %off3, i32 %stride, i32 %loopBound) {
+; GFX12-LABEL: loop_with_epilogue:
+; GFX12: ; %bb.0: ; %entry
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x20
+; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x0
+; GFX12-NEXT: v_and_b32_e32 v3, 0x3ff, v0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_cmp_eq_u32 s9, 0
+; GFX12-NEXT: s_cbranch_scc1 .LBB1_4
+; GFX12-NEXT: ; %bb.1: ; %loop.preheader
+; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v8, 0
+; GFX12-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 0
+; GFX12-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v12, 0
+; GFX12-NEXT: v_dual_mov_b32 v13, 0 :: v_dual_mov_b32 v14, 0
+; GFX12-NEXT: v_dual_mov_b32 v15, 0 :: v_dual_mov_b32 v16, 0
+; GFX12-NEXT: v_dual_mov_b32 v17, 0 :: v_dual_mov_b32 v18, 0
+; GFX12-NEXT: v_dual_mov_b32 v19, 0 :: v_dual_mov_b32 v20, 0
+; GFX12-NEXT: v_dual_mov_b32 v21, 0 :: v_dual_mov_b32 v22, 0
+; GFX12-NEXT: v_mov_b32_e32 v23, 0
+; GFX12-NEXT: .LBB1_2: ; %loop
+; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
+; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_add_nc_u32 v0, s4, v3
+; GFX12-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_add_nc_u32 v4, s5, v3
+; GFX12-NEXT: v_dual_mov_b32 v25, v1 :: v_dual_add_nc_u32 v6, s6, v3
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX12-NEXT: v_lshlrev_b64_e32 v[26:27], 4, v[0:1]
+; GFX12-NEXT: v_add_nc_u32_e32 v24, s7, v3
+; GFX12-NEXT: v_lshlrev_b64_e32 v[4:5], 4, v[4:5]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-NEXT: v_lshlrev_b64_e32 v[6:7], 4, v[6:7]
+; GFX12-NEXT: v_add_nc_u32_e32 v3, s8, v3
+; GFX12-NEXT: s_add_co_i32 s9, s9, -1
+; GFX12-NEXT: v_lshlrev_b64_e32 v[24:25], 4, v[24:25]
+; GFX12-NEXT: v_add_co_u32 v26, vcc_lo, s0, v26
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v27, null, s1, v27, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v28, vcc_lo, s0, v4
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v29, null, s1, v5, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v30, vcc_lo, s0, v6
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v31, null, s1, v7, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v32, vcc_lo, s0, v24
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v33, null, s1, v25, vcc_lo
+; GFX12-NEXT: s_clause 0x3
+; GFX12-NEXT: global_load_b128 v[4:7], v[26:27], off
+; GFX12-NEXT: global_load_b128 v[24:27], v[28:29], off
+; GFX12-NEXT: global_load_b128 v[28:31], v[30:31], off
+; GFX12-NEXT: global_load_b128 v[32:35], v[32:33], off
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: s_cmp_lg_u32 s9, 0
+; GFX12-NEXT: s_wait_loadcnt 0x3
+; GFX12-NEXT: v_add_nc_u32_e32 v20, v4, v20
+; GFX12-NEXT: v_add_nc_u32_e32 v21, v5, v21
+; GFX12-NEXT: v_add_nc_u32_e32 v23, v6, v23
+; GFX12-NEXT: v_add_nc_u32_e32 v22, v7, v22
+; GFX12-NEXT: s_wait_loadcnt 0x2
+; GFX12-NEXT: v_add_nc_u32_e32 v16, v24, v16
+; GFX12-NEXT: v_add_nc_u32_e32 v17, v25, v17
+; GFX12-NEXT: v_add_nc_u32_e32 v18, v26, v18
+; GFX12-NEXT: v_add_nc_u32_e32 v19, v27, v19
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_add_nc_u32_e32 v12, v28, v12
+; GFX12-NEXT: v_add_nc_u32_e32 v13, v29, v13
+; GFX12-NEXT: v_add_nc_u32_e32 v14, v30, v14
+; GFX12-NEXT: v_add_nc_u32_e32 v15, v31, v15
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v8, v32, v8
+; GFX12-NEXT: v_add_nc_u32_e32 v9, v33, v9
+; GFX12-NEXT: v_add_nc_u32_e32 v10, v34, v10
+; GFX12-NEXT: v_add_nc_u32_e32 v11, v35, v11
+; GFX12-NEXT: s_cbranch_scc1 .LBB1_2
+; GFX12-NEXT: ; %bb.3: ; %Flow
+; GFX12-NEXT: v_add_nc_u32_e32 v2, s7, v3
+; GFX12-NEXT: v_add_nc_u32_e32 v4, s6, v3
+; GFX12-NEXT: v_add_nc_u32_e32 v6, s5, v3
+; GFX12-NEXT: v_add_nc_u32_e32 v0, s4, v3
+; GFX12-NEXT: s_branch .LBB1_5
+; GFX12-NEXT: .LBB1_4:
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_dual_mov_b32 v23, 0 :: v_dual_add_nc_u32 v0, s4, v3
+; GFX12-NEXT: v_dual_mov_b32 v21, 0 :: v_dual_add_nc_u32 v6, s5, v3
+; GFX12-NEXT: v_dual_mov_b32 v19, 0 :: v_dual_add_nc_u32 v4, s6, v3
+; GFX12-NEXT: v_dual_mov_b32 v17, 0 :: v_dual_add_nc_u32 v2, s7, v3
+; GFX12-NEXT: v_dual_mov_b32 v22, 0 :: v_dual_mov_b32 v15, 0
+; GFX12-NEXT: v_dual_mov_b32 v20, 0 :: v_dual_mov_b32 v13, 0
+; GFX12-NEXT: v_dual_mov_b32 v18, 0 :: v_dual_mov_b32 v11, 0
+; GFX12-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v9, 0
+; GFX12-NEXT: v_mov_b32_e32 v14, 0
+; GFX12-NEXT: v_mov_b32_e32 v12, 0
+; GFX12-NEXT: v_mov_b32_e32 v10, 0
+; GFX12-NEXT: v_mov_b32_e32 v8, 0
+; GFX12-NEXT: .LBB1_5: ; %epil
+; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX12-NEXT: v_mov_b32_e32 v7, v1
+; GFX12-NEXT: v_lshlrev_b64_e32 v[24:25], 4, v[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v5, v1
+; GFX12-NEXT: v_mov_b32_e32 v3, v1
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT: v_lshlrev_b64_e32 v[6:7], 4, v[6:7]
+; GFX12-NEXT: v_add_co_u32 v24, vcc_lo, s0, v24
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
+; GFX12-NEXT: v_lshlrev_b64_e32 v[26:27], 4, v[4:5]
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v25, null, s1, v25, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, s0, v6
+; GFX12-NEXT: v_lshlrev_b64_e32 v[28:29], 4, v[2:3]
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, s1, v7, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v30, vcc_lo, s0, v26
+; GFX12-NEXT: global_load_b128 v[2:5], v[24:25], off
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v31, null, s1, v27, vcc_lo
+; GFX12-NEXT: global_load_b128 v[24:27], v[6:7], off
+; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, s0, v28
+; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
+; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, s1, v29, vcc_lo
+; GFX12-NEXT: s_clause 0x1
+; GFX12-NEXT: global_load_b128 v[28:31], v[30:31], off
+; GFX12-NEXT: global_load_b128 v[32:35], v[6:7], off
+; GFX12-NEXT: s_wait_loadcnt 0x3
+; GFX12-NEXT: v_add_nc_u32_e32 v5, v5, v22
+; GFX12-NEXT: v_add_nc_u32_e32 v4, v4, v23
+; GFX12-NEXT: v_add_nc_u32_e32 v3, v3, v21
+; GFX12-NEXT: v_add_nc_u32_e32 v2, v2, v20
+; GFX12-NEXT: s_wait_loadcnt 0x2
+; GFX12-NEXT: v_add_nc_u32_e32 v19, v27, v19
+; GFX12-NEXT: v_add_nc_u32_e32 v18, v26, v18
+; GFX12-NEXT: v_add_nc_u32_e32 v17, v25, v17
+; GFX12-NEXT: v_add_nc_u32_e32 v16, v24, v16
+; GFX12-NEXT: s_wait_loadcnt 0x1
+; GFX12-NEXT: v_add_nc_u32_e32 v15, v31, v15
+; GFX12-NEXT: v_add_nc_u32_e32 v14, v30, v14
+; GFX12-NEXT: v_add_nc_u32_e32 v13, v29, v13
+; GFX12-NEXT: v_add_nc_u32_e32 v12, v28, v12
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v11, v35, v11
+; GFX12-NEXT: v_add_nc_u32_e32 v10, v34, v10
+; GFX12-NEXT: v_add_nc_u32_e32 v9, v33, v9
+; GFX12-NEXT: v_add_nc_u32_e32 v8, v32, v8
+; GFX12-NEXT: s_clause 0x3
+; GFX12-NEXT: global_store_b128 v1, v[2:5], s[2:3]
+; GFX12-NEXT: global_store_b128 v1, v[16:19], s[2:3] offset:16
+; GFX12-NEXT: global_store_b128 v1, v[12:15], s[2:3] offset:32
+; GFX12-NEXT: global_store_b128 v1, v[8:11], s[2:3] offset:48
+; GFX12-NEXT: s_endpgm
+entry:
+ %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
+ %idx0.init = add i32 %off0, %tid
+ %idx1.init = add i32 %off1, %tid
+ %idx2.init = add i32 %off2, %tid
+ %idx3.init = add i32 %off3, %tid
+ %isLoopEmpty = icmp eq i32 %loopBound, 0
+ br i1 %isLoopEmpty, label %epil, label %loop
+
+loop:
+ %idx0 = phi i32 [ %idx0.init, %entry ], [ %idx0.next, %loop ]
+ %idx1 = phi i32 [ %idx1.init, %entry ], [ %idx1.next, %loop ]
+ %idx2 = phi i32 [ %idx2.init, %entry ], [ %idx2.next, %loop ]
+ %idx3 = phi i32 [ %idx3.init, %entry ], [ %idx3.next, %loop ]
+ %sum0 = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum0.next, %loop ]
+ %sum1 = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum1.next, %loop ]
+ %sum2 = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum2.next, %loop ]
+ %sum3 = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum3.next, %loop ]
+ %niter = phi i32 [ 0, %entry ], [ %niter.next, %loop ]
+
+ %idx0.64 = zext i32 %idx0 to i64
+ %src0.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx0.64
+ %ld0 = load <4 x i32>, ptr addrspace(1) %src0.p, align 16
+ %sum0.next = add <4 x i32> %ld0, %sum0
+
+ %idx1.64 = zext i32 %idx1 to i64
+ %src1.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx1.64
+ %ld1 = load <4 x i32>, ptr addrspace(1) %src1.p, align 16
+ %sum1.next = add <4 x i32> %ld1, %sum1
+
+ %idx2.64 = zext i32 %idx2 to i64
+ %src2.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx2.64
+ %ld2 = load <4 x i32>, ptr addrspace(1) %src2.p, align 16
+ %sum2.next = add <4 x i32> %ld2, %sum2
+
+ %idx3.64 = zext i32 %idx3 to i64
+ %src3.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx3.64
+ %ld3 = load <4 x i32>, ptr addrspace(1) %src3.p, align 16
+ %sum3.next = add <4 x i32> %ld3, %sum3
+
+ %idx0.next = add i32 %idx0, %stride
+ %idx1.next = add i32 %idx1, %stride
+ %idx2.next = add i32 %idx2, %stride
+ %idx3.next = add i32 %idx3, %stride
+ %niter.next = add i32 %niter, 1
+ %done = icmp eq i32 %niter.next, %loopBound
+ br i1 %done, label %epil, label %loop
+
+epil:
+ %idx0.epil = phi i32 [ %idx0.init, %entry ], [ %idx0.next, %loop ]
+ %idx1.epil = phi i32 [ %idx1.init, %entry ], [ %idx1.next, %loop ]
+ %idx2.epil = phi i32 [ %idx2.init, %entry ], [ %idx2.next, %loop ]
+ %idx3.epil = phi i32 [ %idx3.init, %entry ], [ %idx3.next, %loop ]
+ %sum0.epil = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum0.next, %loop ]
+ %sum1.epil = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum1.next, %loop ]
+ %sum2.epil = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum2.next, %loop ]
+ %sum3.epil = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum3.next, %loop ]
+
+ %idx0.epil.64 = zext i32 %idx0.epil to i64
+ %src0.epil.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx0.epil.64
+ %ld0.epil = load <4 x i32>, ptr addrspace(1) %src0.epil.p, align 16
+ %sum0.epil.next = add <4 x i32> %ld0.epil, %sum0.epil
+
+ %idx1.epil.64 = zext i32 %idx1.epil to i64
+ %src1.epil.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx1.epil.64
+ %ld1.epil = load <4 x i32>, ptr addrspace(1) %src1.epil.p, align 16
+ %sum1.epil.next = add <4 x i32> %ld1.epil, %sum1.epil
+
+ %idx2.epil.64 = zext i32 %idx2.epil to i64
+ %src2.epil.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx2.epil.64
+ %ld2.epil = load <4 x i32>, ptr addrspace(1) %src2.epil.p, align 16
+ %sum2.epil.next = add <4 x i32> %ld2.epil, %sum2.epil
+
+ %idx3.epil.64 = zext i32 %idx3.epil to i64
+ %src3.epil.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx3.epil.64
+ %ld3.epil = load <4 x i32>, ptr addrspace(1) %src3.epil.p, align 16
+ %sum3.epil.next = add <4 x i32> %ld3.epil, %sum3.epil
+
+ %dst0.p = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 0
+ %dst1.p = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 1
+ %dst2.p = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 2
+ %dst3.p = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 3
+ store <4 x i32> %sum0.epil.next, ptr addrspace(1) %dst0.p, align 16
+ store <4 x i32> %sum1.epil.next, ptr addrspace(1) %dst1.p, align 16
+ store <4 x i32> %sum2.epil.next, ptr addrspace(1) %dst2.p, align 16
+ store <4 x i32> %sum3.epil.next, ptr addrspace(1) %dst3.p, align 16
+ ret void
+}
>From 0128764a69eb0993b952905349473ef3bbc344b7 Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Fri, 5 Jun 2026 16:29:44 -0500
Subject: [PATCH 4/7] Use lit test from #201943
Copied verbatim
---
.../AMDGPU/consecutive-loads-in-branch.ll | 445 +++++-------------
1 file changed, 105 insertions(+), 340 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/consecutive-loads-in-branch.ll b/llvm/test/CodeGen/AMDGPU/consecutive-loads-in-branch.ll
index a9d2c876102b0..8947a576bb3b5 100644
--- a/llvm/test/CodeGen/AMDGPU/consecutive-loads-in-branch.ll
+++ b/llvm/test/CodeGen/AMDGPU/consecutive-loads-in-branch.ll
@@ -1,374 +1,139 @@
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 6
; RUN: llc -mtriple=amdgcn-amd-amdhsa -mcpu=gfx1200 < %s | FileCheck -check-prefix=GFX12 %s
-define amdgpu_kernel void @straightline(ptr addrspace(1) noalias %inBuf, ptr addrspace(1) noalias %outBuf, i32 %A.base, i32 %B.base, i32 %C.base, i32 %D.base, i32 %stride, i1 %cond) {
-; GFX12-LABEL: straightline:
+define amdgpu_kernel void @straightline_kernel(ptr addrspace(1) noalias %in, ptr addrspace(1) noalias %out, i32 %base, i1 %cond) {
+; GFX12-LABEL: straightline_kernel:
; GFX12: ; %bb.0: ; %entry
; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b256 s[8:15], s[4:5], 0x0
-; GFX12-NEXT: s_load_b64 s[4:5], s[4:5], 0x20
-; GFX12-NEXT: v_and_b32_e32 v1, 0x3ff, v0
-; GFX12-NEXT: s_mov_b32 s0, 0
+; GFX12-NEXT: s_load_b64 s[6:7], s[4:5], 0x10
+; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x0
+; GFX12-NEXT: v_and_b32_e32 v0, 0x3ff, v0
+; GFX12-NEXT: s_mov_b32 s4, 0
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_add_nc_u32_e32 v0, s12, v1
-; GFX12-NEXT: v_add_nc_u32_e32 v2, s13, v1
-; GFX12-NEXT: v_add_nc_u32_e32 v4, s14, v1
-; GFX12-NEXT: v_add_nc_u32_e32 v6, s15, v1
-; GFX12-NEXT: s_bitcmp0_b32 s5, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_nc_u32_e32 v0, s6, v0
+; GFX12-NEXT: s_bitcmp0_b32 s7, 0
+; GFX12-NEXT: v_add_nc_u32_e32 v2, 64, v0
+; GFX12-NEXT: v_add_nc_u32_e32 v4, 0x80, v0
+; GFX12-NEXT: v_add_nc_u32_e32 v6, 0xc0, v0
; GFX12-NEXT: s_cbranch_scc1 .LBB0_2
-; GFX12-NEXT: ; %bb.1: ; %fake.pred
-; GFX12-NEXT: s_load_b128 s[0:3], s[8:9], 0x0
-; GFX12-NEXT: v_add_nc_u32_e32 v0, s4, v0
-; GFX12-NEXT: v_add_nc_u32_e32 v2, s4, v2
-; GFX12-NEXT: v_add_nc_u32_e32 v4, s4, v4
-; GFX12-NEXT: v_add_nc_u32_e32 v6, s4, v6
+; GFX12-NEXT: ; %bb.1: ; %pred
+; GFX12-NEXT: s_load_b128 s[4:7], s[0:1], 0x0
+; GFX12-NEXT: v_add_nc_u32_e32 v0, 1, v0
+; GFX12-NEXT: v_add_nc_u32_e32 v2, 1, v2
+; GFX12-NEXT: v_add_nc_u32_e32 v4, 1, v4
+; GFX12-NEXT: v_add_nc_u32_e32 v6, 1, v6
; GFX12-NEXT: s_branch .LBB0_3
; GFX12-NEXT: .LBB0_2:
-; GFX12-NEXT: s_mov_b32 s1, 0
-; GFX12-NEXT: s_mov_b32 s2, 0
-; GFX12-NEXT: s_mov_b32 s3, 0
+; GFX12-NEXT: s_mov_b32 s5, 0
+; GFX12-NEXT: s_mov_b32 s6, 0
+; GFX12-NEXT: s_mov_b32 s7, 0
; GFX12-NEXT: .LBB0_3: ; %main
; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_lshlrev_b64_e32 v[8:9], 4, v[0:1]
; GFX12-NEXT: v_mov_b32_e32 v3, v1
; GFX12-NEXT: v_mov_b32_e32 v5, v1
-; GFX12-NEXT: v_lshlrev_b64_e32 v[8:9], 4, v[0:1]
-; GFX12-NEXT: v_mov_b32_e32 v7, v1
; GFX12-NEXT: v_lshlrev_b64_e32 v[10:11], 4, v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX12-NEXT: v_add_co_u32 v8, vcc_lo, s0, v8
+; GFX12-NEXT: v_add_co_ci_u32_e64 v9, null, s1, v9, vcc_lo
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-NEXT: v_lshlrev_b64_e32 v[12:13], 4, v[4:5]
-; GFX12-NEXT: v_add_co_u32 v8, vcc_lo, s8, v8
+; GFX12-NEXT: global_load_b128 v[2:5], v[8:9], off
+; GFX12-NEXT: s_wait_loadcnt 0x0
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_add_nc_u32 v4, s6, v4
+; GFX12-NEXT: v_add_nc_u32_e32 v5, s7, v5
+; GFX12-NEXT: v_add_nc_u32_e32 v3, s5, v3
+; GFX12-NEXT: v_add_nc_u32_e32 v2, s4, v2
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-NEXT: v_lshlrev_b64_e32 v[14:15], 4, v[6:7]
-; GFX12-NEXT: v_add_co_ci_u32_e64 v9, null, s9, v9, vcc_lo
-; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, s8, v10
+; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, s0, v10
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, s9, v11, vcc_lo
-; GFX12-NEXT: v_add_co_u32 v10, vcc_lo, s8, v12
+; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, s1, v11, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v10, vcc_lo, s0, v12
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v11, null, s9, v13, vcc_lo
-; GFX12-NEXT: v_add_co_u32 v14, vcc_lo, s8, v14
+; GFX12-NEXT: v_add_co_ci_u32_e64 v11, null, s1, v13, vcc_lo
+; GFX12-NEXT: v_add_co_u32 v14, vcc_lo, s0, v14
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v15, null, s9, v15, vcc_lo
-; GFX12-NEXT: s_clause 0x3
-; GFX12-NEXT: global_load_b128 v[2:5], v[8:9], off
+; GFX12-NEXT: v_add_co_ci_u32_e64 v15, null, s1, v15, vcc_lo
+; GFX12-NEXT: s_clause 0x2
; GFX12-NEXT: global_load_b128 v[6:9], v[6:7], off
; GFX12-NEXT: global_load_b128 v[10:13], v[10:11], off
; GFX12-NEXT: global_load_b128 v[14:17], v[14:15], off
-; GFX12-NEXT: s_wait_loadcnt 0x3
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: v_add_nc_u32_e32 v5, s3, v5
-; GFX12-NEXT: v_add_nc_u32_e32 v4, s2, v4
-; GFX12-NEXT: v_add_nc_u32_e32 v3, s1, v3
-; GFX12-NEXT: v_add_nc_u32_e32 v2, s0, v2
-; GFX12-NEXT: s_wait_loadcnt 0x2
-; GFX12-NEXT: v_add_nc_u32_e32 v9, s3, v9
-; GFX12-NEXT: v_add_nc_u32_e32 v8, s2, v8
-; GFX12-NEXT: v_add_nc_u32_e32 v7, s1, v7
-; GFX12-NEXT: v_add_nc_u32_e32 v6, s0, v6
-; GFX12-NEXT: s_wait_loadcnt 0x1
-; GFX12-NEXT: v_add_nc_u32_e32 v13, s3, v13
-; GFX12-NEXT: v_add_nc_u32_e32 v12, s2, v12
-; GFX12-NEXT: v_add_nc_u32_e32 v11, s1, v11
-; GFX12-NEXT: v_add_nc_u32_e32 v10, s0, v10
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_nc_u32_e32 v17, s3, v17
-; GFX12-NEXT: v_add_nc_u32_e32 v16, s2, v16
-; GFX12-NEXT: v_add_nc_u32_e32 v15, s1, v15
-; GFX12-NEXT: v_add_nc_u32_e32 v14, s0, v14
-; GFX12-NEXT: s_clause 0x3
-; GFX12-NEXT: global_store_b128 v1, v[2:5], s[10:11]
-; GFX12-NEXT: global_store_b128 v1, v[6:9], s[10:11] offset:16
-; GFX12-NEXT: global_store_b128 v1, v[10:13], s[10:11] offset:32
-; GFX12-NEXT: global_store_b128 v1, v[14:17], s[10:11] offset:48
-; GFX12-NEXT: s_endpgm
-entry:
- %tid = tail call i32 @llvm.amdgcn.workitem.id.x()
- %A.idx.start = add i32 %A.base, %tid
- %B.idx.start = add i32 %B.base, %tid
- %C.idx.start = add i32 %C.base, %tid
- %D.idx.start = add i32 %D.base, %tid
- br i1 %cond, label %fake.pred, label %main
-
-fake.pred: ; preds = %entry
- %A.idx.next = add i32 %A.idx.start, %stride
- %B.idx.next = add i32 %B.idx.start, %stride
- %C.idx.next = add i32 %C.idx.start, %stride
- %D.idx.next = add i32 %D.idx.start, %stride
- %seed.ptr = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 0
- %seed.acc = load <4 x i32>, ptr addrspace(1) %seed.ptr, align 16
- br label %main
-
-main: ; preds = %fake.pred, %entry
- %acc = phi <4 x i32> [ zeroinitializer, %entry ], [ %seed.acc, %fake.pred ]
- %A.idx = phi i32 [ %A.idx.start, %entry ], [ %A.idx.next, %fake.pred ]
- %B.idx = phi i32 [ %B.idx.start, %entry ], [ %B.idx.next, %fake.pred ]
- %C.idx = phi i32 [ %C.idx.start, %entry ], [ %C.idx.next, %fake.pred ]
- %D.idx = phi i32 [ %D.idx.start, %entry ], [ %D.idx.next, %fake.pred ]
-
- %A.idx.64 = zext i32 %A.idx to i64
- %A.ptr = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %A.idx.64
- %A.data = load <4 x i32>, ptr addrspace(1) %A.ptr, align 16
- %A.acc = add <4 x i32> %A.data, %acc
- %B.idx.64 = zext i32 %B.idx to i64
- %B.ptr = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %B.idx.64
- %B.data = load <4 x i32>, ptr addrspace(1) %B.ptr, align 16
- %B.acc = add <4 x i32> %B.data, %acc
- %C.idx.64 = zext i32 %C.idx to i64
- %C.ptr = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %C.idx.64
- %C.data = load <4 x i32>, ptr addrspace(1) %C.ptr, align 16
- %C.acc = add <4 x i32> %C.data, %acc
- %D.idx.64 = zext i32 %D.idx to i64
- %D.ptr = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %D.idx.64
- %D.data = load <4 x i32>, ptr addrspace(1) %D.ptr, align 16
- %D.acc = add <4 x i32> %D.data, %acc
-
- %A.out = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 0
- %B.out = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 1
- %C.out = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 2
- %D.out = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 3
- store <4 x i32> %A.acc, ptr addrspace(1) %A.out, align 16
- store <4 x i32> %B.acc, ptr addrspace(1) %B.out, align 16
- store <4 x i32> %C.acc, ptr addrspace(1) %C.out, align 16
- store <4 x i32> %D.acc, ptr addrspace(1) %D.out, align 16
- ret void
-}
-
-define amdgpu_kernel void @loop_with_epilogue(ptr addrspace(1) noalias %inBuf, ptr addrspace(1) noalias %outBuf, i32 %off0, i32 %off1, i32 %off2, i32 %off3, i32 %stride, i32 %loopBound) {
-; GFX12-LABEL: loop_with_epilogue:
-; GFX12: ; %bb.0: ; %entry
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: s_load_b64 s[8:9], s[4:5], 0x20
-; GFX12-NEXT: s_load_b256 s[0:7], s[4:5], 0x0
-; GFX12-NEXT: v_and_b32_e32 v3, 0x3ff, v0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_cmp_eq_u32 s9, 0
-; GFX12-NEXT: s_cbranch_scc1 .LBB1_4
-; GFX12-NEXT: ; %bb.1: ; %loop.preheader
-; GFX12-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_mov_b32 v8, 0
-; GFX12-NEXT: v_dual_mov_b32 v9, 0 :: v_dual_mov_b32 v10, 0
-; GFX12-NEXT: v_dual_mov_b32 v11, 0 :: v_dual_mov_b32 v12, 0
-; GFX12-NEXT: v_dual_mov_b32 v13, 0 :: v_dual_mov_b32 v14, 0
-; GFX12-NEXT: v_dual_mov_b32 v15, 0 :: v_dual_mov_b32 v16, 0
-; GFX12-NEXT: v_dual_mov_b32 v17, 0 :: v_dual_mov_b32 v18, 0
-; GFX12-NEXT: v_dual_mov_b32 v19, 0 :: v_dual_mov_b32 v20, 0
-; GFX12-NEXT: v_dual_mov_b32 v21, 0 :: v_dual_mov_b32 v22, 0
-; GFX12-NEXT: v_mov_b32_e32 v23, 0
-; GFX12-NEXT: .LBB1_2: ; %loop
-; GFX12-NEXT: ; =>This Inner Loop Header: Depth=1
-; GFX12-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_add_nc_u32 v0, s4, v3
-; GFX12-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_add_nc_u32 v4, s5, v3
-; GFX12-NEXT: v_dual_mov_b32 v25, v1 :: v_dual_add_nc_u32 v6, s6, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_lshlrev_b64_e32 v[26:27], 4, v[0:1]
-; GFX12-NEXT: v_add_nc_u32_e32 v24, s7, v3
-; GFX12-NEXT: v_lshlrev_b64_e32 v[4:5], 4, v[4:5]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4)
-; GFX12-NEXT: v_lshlrev_b64_e32 v[6:7], 4, v[6:7]
-; GFX12-NEXT: v_add_nc_u32_e32 v3, s8, v3
-; GFX12-NEXT: s_add_co_i32 s9, s9, -1
-; GFX12-NEXT: v_lshlrev_b64_e32 v[24:25], 4, v[24:25]
-; GFX12-NEXT: v_add_co_u32 v26, vcc_lo, s0, v26
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v27, null, s1, v27, vcc_lo
-; GFX12-NEXT: v_add_co_u32 v28, vcc_lo, s0, v4
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v29, null, s1, v5, vcc_lo
-; GFX12-NEXT: v_add_co_u32 v30, vcc_lo, s0, v6
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v31, null, s1, v7, vcc_lo
-; GFX12-NEXT: v_add_co_u32 v32, vcc_lo, s0, v24
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v33, null, s1, v25, vcc_lo
-; GFX12-NEXT: s_clause 0x3
-; GFX12-NEXT: global_load_b128 v[4:7], v[26:27], off
-; GFX12-NEXT: global_load_b128 v[24:27], v[28:29], off
-; GFX12-NEXT: global_load_b128 v[28:31], v[30:31], off
-; GFX12-NEXT: global_load_b128 v[32:35], v[32:33], off
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: s_cmp_lg_u32 s9, 0
-; GFX12-NEXT: s_wait_loadcnt 0x3
-; GFX12-NEXT: v_add_nc_u32_e32 v20, v4, v20
-; GFX12-NEXT: v_add_nc_u32_e32 v21, v5, v21
-; GFX12-NEXT: v_add_nc_u32_e32 v23, v6, v23
-; GFX12-NEXT: v_add_nc_u32_e32 v22, v7, v22
; GFX12-NEXT: s_wait_loadcnt 0x2
-; GFX12-NEXT: v_add_nc_u32_e32 v16, v24, v16
-; GFX12-NEXT: v_add_nc_u32_e32 v17, v25, v17
-; GFX12-NEXT: v_add_nc_u32_e32 v18, v26, v18
-; GFX12-NEXT: v_add_nc_u32_e32 v19, v27, v19
-; GFX12-NEXT: s_wait_loadcnt 0x1
-; GFX12-NEXT: v_add_nc_u32_e32 v12, v28, v12
-; GFX12-NEXT: v_add_nc_u32_e32 v13, v29, v13
-; GFX12-NEXT: v_add_nc_u32_e32 v14, v30, v14
-; GFX12-NEXT: v_add_nc_u32_e32 v15, v31, v15
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_nc_u32_e32 v8, v32, v8
-; GFX12-NEXT: v_add_nc_u32_e32 v9, v33, v9
-; GFX12-NEXT: v_add_nc_u32_e32 v10, v34, v10
-; GFX12-NEXT: v_add_nc_u32_e32 v11, v35, v11
-; GFX12-NEXT: s_cbranch_scc1 .LBB1_2
-; GFX12-NEXT: ; %bb.3: ; %Flow
-; GFX12-NEXT: v_add_nc_u32_e32 v2, s7, v3
-; GFX12-NEXT: v_add_nc_u32_e32 v4, s6, v3
-; GFX12-NEXT: v_add_nc_u32_e32 v6, s5, v3
-; GFX12-NEXT: v_add_nc_u32_e32 v0, s4, v3
-; GFX12-NEXT: s_branch .LBB1_5
-; GFX12-NEXT: .LBB1_4:
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
-; GFX12-NEXT: v_dual_mov_b32 v23, 0 :: v_dual_add_nc_u32 v0, s4, v3
-; GFX12-NEXT: v_dual_mov_b32 v21, 0 :: v_dual_add_nc_u32 v6, s5, v3
-; GFX12-NEXT: v_dual_mov_b32 v19, 0 :: v_dual_add_nc_u32 v4, s6, v3
-; GFX12-NEXT: v_dual_mov_b32 v17, 0 :: v_dual_add_nc_u32 v2, s7, v3
-; GFX12-NEXT: v_dual_mov_b32 v22, 0 :: v_dual_mov_b32 v15, 0
-; GFX12-NEXT: v_dual_mov_b32 v20, 0 :: v_dual_mov_b32 v13, 0
-; GFX12-NEXT: v_dual_mov_b32 v18, 0 :: v_dual_mov_b32 v11, 0
-; GFX12-NEXT: v_dual_mov_b32 v16, 0 :: v_dual_mov_b32 v9, 0
-; GFX12-NEXT: v_mov_b32_e32 v14, 0
-; GFX12-NEXT: v_mov_b32_e32 v12, 0
-; GFX12-NEXT: v_mov_b32_e32 v10, 0
-; GFX12-NEXT: v_mov_b32_e32 v8, 0
-; GFX12-NEXT: .LBB1_5: ; %epil
-; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_mov_b32_e32 v7, v1
-; GFX12-NEXT: v_lshlrev_b64_e32 v[24:25], 4, v[0:1]
-; GFX12-NEXT: v_mov_b32_e32 v5, v1
-; GFX12-NEXT: v_mov_b32_e32 v3, v1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_lshlrev_b64_e32 v[6:7], 4, v[6:7]
-; GFX12-NEXT: v_add_co_u32 v24, vcc_lo, s0, v24
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_4)
-; GFX12-NEXT: v_lshlrev_b64_e32 v[26:27], 4, v[4:5]
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v25, null, s1, v25, vcc_lo
-; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, s0, v6
-; GFX12-NEXT: v_lshlrev_b64_e32 v[28:29], 4, v[2:3]
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, s1, v7, vcc_lo
-; GFX12-NEXT: v_add_co_u32 v30, vcc_lo, s0, v26
-; GFX12-NEXT: global_load_b128 v[2:5], v[24:25], off
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v31, null, s1, v27, vcc_lo
-; GFX12-NEXT: global_load_b128 v[24:27], v[6:7], off
-; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, s0, v28
-; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
-; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, s1, v29, vcc_lo
-; GFX12-NEXT: s_clause 0x1
-; GFX12-NEXT: global_load_b128 v[28:31], v[30:31], off
-; GFX12-NEXT: global_load_b128 v[32:35], v[6:7], off
-; GFX12-NEXT: s_wait_loadcnt 0x3
-; GFX12-NEXT: v_add_nc_u32_e32 v5, v5, v22
-; GFX12-NEXT: v_add_nc_u32_e32 v4, v4, v23
-; GFX12-NEXT: v_add_nc_u32_e32 v3, v3, v21
-; GFX12-NEXT: v_add_nc_u32_e32 v2, v2, v20
-; GFX12-NEXT: s_wait_loadcnt 0x2
-; GFX12-NEXT: v_add_nc_u32_e32 v19, v27, v19
-; GFX12-NEXT: v_add_nc_u32_e32 v18, v26, v18
-; GFX12-NEXT: v_add_nc_u32_e32 v17, v25, v17
-; GFX12-NEXT: v_add_nc_u32_e32 v16, v24, v16
+; GFX12-NEXT: v_add_nc_u32_e32 v9, s7, v9
+; GFX12-NEXT: v_add_nc_u32_e32 v8, s6, v8
+; GFX12-NEXT: v_add_nc_u32_e32 v7, s5, v7
+; GFX12-NEXT: v_add_nc_u32_e32 v6, s4, v6
; GFX12-NEXT: s_wait_loadcnt 0x1
-; GFX12-NEXT: v_add_nc_u32_e32 v15, v31, v15
-; GFX12-NEXT: v_add_nc_u32_e32 v14, v30, v14
-; GFX12-NEXT: v_add_nc_u32_e32 v13, v29, v13
-; GFX12-NEXT: v_add_nc_u32_e32 v12, v28, v12
+; GFX12-NEXT: v_add_nc_u32_e32 v13, s7, v13
+; GFX12-NEXT: v_add_nc_u32_e32 v12, s6, v12
+; GFX12-NEXT: v_add_nc_u32_e32 v11, s5, v11
+; GFX12-NEXT: v_add_nc_u32_e32 v10, s4, v10
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_add_nc_u32_e32 v11, v35, v11
-; GFX12-NEXT: v_add_nc_u32_e32 v10, v34, v10
-; GFX12-NEXT: v_add_nc_u32_e32 v9, v33, v9
-; GFX12-NEXT: v_add_nc_u32_e32 v8, v32, v8
+; GFX12-NEXT: v_add_nc_u32_e32 v17, s7, v17
+; GFX12-NEXT: v_add_nc_u32_e32 v16, s6, v16
+; GFX12-NEXT: v_add_nc_u32_e32 v15, s5, v15
+; GFX12-NEXT: v_add_nc_u32_e32 v14, s4, v14
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v1, v[2:5], s[2:3]
-; GFX12-NEXT: global_store_b128 v1, v[16:19], s[2:3] offset:16
-; GFX12-NEXT: global_store_b128 v1, v[12:15], s[2:3] offset:32
-; GFX12-NEXT: global_store_b128 v1, v[8:11], s[2:3] offset:48
+; GFX12-NEXT: global_store_b128 v1, v[6:9], s[2:3] offset:16
+; GFX12-NEXT: global_store_b128 v1, v[10:13], s[2:3] offset:32
+; GFX12-NEXT: global_store_b128 v1, v[14:17], s[2:3] offset:48
; GFX12-NEXT: s_endpgm
entry:
%tid = tail call i32 @llvm.amdgcn.workitem.id.x()
- %idx0.init = add i32 %off0, %tid
- %idx1.init = add i32 %off1, %tid
- %idx2.init = add i32 %off2, %tid
- %idx3.init = add i32 %off3, %tid
- %isLoopEmpty = icmp eq i32 %loopBound, 0
- br i1 %isLoopEmpty, label %epil, label %loop
-
-loop:
- %idx0 = phi i32 [ %idx0.init, %entry ], [ %idx0.next, %loop ]
- %idx1 = phi i32 [ %idx1.init, %entry ], [ %idx1.next, %loop ]
- %idx2 = phi i32 [ %idx2.init, %entry ], [ %idx2.next, %loop ]
- %idx3 = phi i32 [ %idx3.init, %entry ], [ %idx3.next, %loop ]
- %sum0 = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum0.next, %loop ]
- %sum1 = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum1.next, %loop ]
- %sum2 = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum2.next, %loop ]
- %sum3 = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum3.next, %loop ]
- %niter = phi i32 [ 0, %entry ], [ %niter.next, %loop ]
-
- %idx0.64 = zext i32 %idx0 to i64
- %src0.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx0.64
- %ld0 = load <4 x i32>, ptr addrspace(1) %src0.p, align 16
- %sum0.next = add <4 x i32> %ld0, %sum0
-
- %idx1.64 = zext i32 %idx1 to i64
- %src1.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx1.64
- %ld1 = load <4 x i32>, ptr addrspace(1) %src1.p, align 16
- %sum1.next = add <4 x i32> %ld1, %sum1
-
- %idx2.64 = zext i32 %idx2 to i64
- %src2.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx2.64
- %ld2 = load <4 x i32>, ptr addrspace(1) %src2.p, align 16
- %sum2.next = add <4 x i32> %ld2, %sum2
-
- %idx3.64 = zext i32 %idx3 to i64
- %src3.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx3.64
- %ld3 = load <4 x i32>, ptr addrspace(1) %src3.p, align 16
- %sum3.next = add <4 x i32> %ld3, %sum3
-
- %idx0.next = add i32 %idx0, %stride
- %idx1.next = add i32 %idx1, %stride
- %idx2.next = add i32 %idx2, %stride
- %idx3.next = add i32 %idx3, %stride
- %niter.next = add i32 %niter, 1
- %done = icmp eq i32 %niter.next, %loopBound
- br i1 %done, label %epil, label %loop
-
-epil:
- %idx0.epil = phi i32 [ %idx0.init, %entry ], [ %idx0.next, %loop ]
- %idx1.epil = phi i32 [ %idx1.init, %entry ], [ %idx1.next, %loop ]
- %idx2.epil = phi i32 [ %idx2.init, %entry ], [ %idx2.next, %loop ]
- %idx3.epil = phi i32 [ %idx3.init, %entry ], [ %idx3.next, %loop ]
- %sum0.epil = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum0.next, %loop ]
- %sum1.epil = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum1.next, %loop ]
- %sum2.epil = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum2.next, %loop ]
- %sum3.epil = phi <4 x i32> [ zeroinitializer, %entry ], [ %sum3.next, %loop ]
-
- %idx0.epil.64 = zext i32 %idx0.epil to i64
- %src0.epil.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx0.epil.64
- %ld0.epil = load <4 x i32>, ptr addrspace(1) %src0.epil.p, align 16
- %sum0.epil.next = add <4 x i32> %ld0.epil, %sum0.epil
-
- %idx1.epil.64 = zext i32 %idx1.epil to i64
- %src1.epil.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx1.epil.64
- %ld1.epil = load <4 x i32>, ptr addrspace(1) %src1.epil.p, align 16
- %sum1.epil.next = add <4 x i32> %ld1.epil, %sum1.epil
-
- %idx2.epil.64 = zext i32 %idx2.epil to i64
- %src2.epil.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx2.epil.64
- %ld2.epil = load <4 x i32>, ptr addrspace(1) %src2.epil.p, align 16
- %sum2.epil.next = add <4 x i32> %ld2.epil, %sum2.epil
-
- %idx3.epil.64 = zext i32 %idx3.epil to i64
- %src3.epil.p = getelementptr inbounds nuw [16 x i8], ptr addrspace(1) %inBuf, i64 %idx3.epil.64
- %ld3.epil = load <4 x i32>, ptr addrspace(1) %src3.epil.p, align 16
- %sum3.epil.next = add <4 x i32> %ld3.epil, %sum3.epil
+ %A.idx.0 = add i32 %base, %tid
+ %B.idx.0 = add i32 %A.idx.0, 64
+ %C.idx.0 = add i32 %A.idx.0, 128
+ %D.idx.0 = add i32 %A.idx.0, 192
+ br i1 %cond, label %pred, label %main
+
+pred: ; preds = %entry
+ %A.idx.1 = add i32 %A.idx.0, 1
+ %B.idx.1 = add i32 %B.idx.0, 1
+ %C.idx.1 = add i32 %C.idx.0, 1
+ %D.idx.1 = add i32 %D.idx.0, 1
+ %seed = load <4 x i32>, ptr addrspace(1) %in, align 16
+ br label %main
- %dst0.p = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 0
- %dst1.p = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 1
- %dst2.p = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 2
- %dst3.p = getelementptr inbounds [16 x i8], ptr addrspace(1) %outBuf, i64 3
- store <4 x i32> %sum0.epil.next, ptr addrspace(1) %dst0.p, align 16
- store <4 x i32> %sum1.epil.next, ptr addrspace(1) %dst1.p, align 16
- store <4 x i32> %sum2.epil.next, ptr addrspace(1) %dst2.p, align 16
- store <4 x i32> %sum3.epil.next, ptr addrspace(1) %dst3.p, align 16
+main: ; preds = %pred, %entry
+ %acc = phi <4 x i32> [ zeroinitializer, %entry ], [ %seed, %pred ]
+ %A.idx = phi i32 [ %A.idx.0, %entry ], [ %A.idx.1, %pred ]
+ %B.idx = phi i32 [ %B.idx.0, %entry ], [ %B.idx.1, %pred ]
+ %C.idx = phi i32 [ %C.idx.0, %entry ], [ %C.idx.1, %pred ]
+ %D.idx = phi i32 [ %D.idx.0, %entry ], [ %D.idx.1, %pred ]
+
+ %A.i64 = zext i32 %A.idx to i64
+ %A.ptr = getelementptr inbounds [16 x i8], ptr addrspace(1) %in, i64 %A.i64
+ %A.load = load <4 x i32>, ptr addrspace(1) %A.ptr, align 16
+ %aa = add <4 x i32> %A.load, %acc
+
+ %B.i64 = zext i32 %B.idx to i64
+ %B.ptr = getelementptr inbounds [16 x i8], ptr addrspace(1) %in, i64 %B.i64
+ %B.load = load <4 x i32>, ptr addrspace(1) %B.ptr, align 16
+ %ba = add <4 x i32> %B.load, %acc
+
+ %C.i64 = zext i32 %C.idx to i64
+ %C.ptr = getelementptr inbounds [16 x i8], ptr addrspace(1) %in, i64 %C.i64
+ %C.load = load <4 x i32>, ptr addrspace(1) %C.ptr, align 16
+ %ca = add <4 x i32> %C.load, %acc
+
+ %D.i64 = zext i32 %D.idx to i64
+ %D.ptr = getelementptr inbounds [16 x i8], ptr addrspace(1) %in, i64 %D.i64
+ %D.load = load <4 x i32>, ptr addrspace(1) %D.ptr, align 16
+ %da = add <4 x i32> %D.load, %acc
+
+ %A.out = getelementptr inbounds [16 x i8], ptr addrspace(1) %out, i64 0
+ %B.out = getelementptr inbounds [16 x i8], ptr addrspace(1) %out, i64 1
+ %C.out = getelementptr inbounds [16 x i8], ptr addrspace(1) %out, i64 2
+ %D.out = getelementptr inbounds [16 x i8], ptr addrspace(1) %out, i64 3
+ store <4 x i32> %aa, ptr addrspace(1) %A.out, align 16
+ store <4 x i32> %ba, ptr addrspace(1) %B.out, align 16
+ store <4 x i32> %ca, ptr addrspace(1) %C.out, align 16
+ store <4 x i32> %da, ptr addrspace(1) %D.out, align 16
ret void
}
>From 41eeb87c33e85463935cf88e6fc3e5272f23a0e3 Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Fri, 5 Jun 2026 16:31:08 -0500
Subject: [PATCH 5/7] Updated test case to show behaviour diff
---
.../AMDGPU/consecutive-loads-in-branch.ll | 30 +++++++++----------
1 file changed, 15 insertions(+), 15 deletions(-)
diff --git a/llvm/test/CodeGen/AMDGPU/consecutive-loads-in-branch.ll b/llvm/test/CodeGen/AMDGPU/consecutive-loads-in-branch.ll
index 8947a576bb3b5..f9f481f56fa7c 100644
--- a/llvm/test/CodeGen/AMDGPU/consecutive-loads-in-branch.ll
+++ b/llvm/test/CodeGen/AMDGPU/consecutive-loads-in-branch.ll
@@ -30,26 +30,18 @@ define amdgpu_kernel void @straightline_kernel(ptr addrspace(1) noalias %in, ptr
; GFX12-NEXT: s_mov_b32 s7, 0
; GFX12-NEXT: .LBB0_3: ; %main
; GFX12-NEXT: v_mov_b32_e32 v1, 0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_lshlrev_b64_e32 v[8:9], 4, v[0:1]
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX12-NEXT: v_mov_b32_e32 v3, v1
; GFX12-NEXT: v_mov_b32_e32 v5, v1
+; GFX12-NEXT: v_lshlrev_b64_e32 v[8:9], 4, v[0:1]
+; GFX12-NEXT: v_mov_b32_e32 v7, v1
; GFX12-NEXT: v_lshlrev_b64_e32 v[10:11], 4, v[2:3]
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX12-NEXT: v_add_co_u32 v8, vcc_lo, s0, v8
-; GFX12-NEXT: v_add_co_ci_u32_e64 v9, null, s1, v9, vcc_lo
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-NEXT: v_lshlrev_b64_e32 v[12:13], 4, v[4:5]
-; GFX12-NEXT: global_load_b128 v[2:5], v[8:9], off
-; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_add_nc_u32 v4, s6, v4
-; GFX12-NEXT: v_add_nc_u32_e32 v5, s7, v5
-; GFX12-NEXT: v_add_nc_u32_e32 v3, s5, v3
-; GFX12-NEXT: v_add_nc_u32_e32 v2, s4, v2
+; GFX12-NEXT: v_add_co_u32 v8, vcc_lo, s0, v8
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-NEXT: v_lshlrev_b64_e32 v[14:15], 4, v[6:7]
+; GFX12-NEXT: v_add_co_ci_u32_e64 v9, null, s1, v9, vcc_lo
; GFX12-NEXT: v_add_co_u32 v6, vcc_lo, s0, v10
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_add_co_ci_u32_e64 v7, null, s1, v11, vcc_lo
@@ -59,10 +51,18 @@ define amdgpu_kernel void @straightline_kernel(ptr addrspace(1) noalias %in, ptr
; GFX12-NEXT: v_add_co_u32 v14, vcc_lo, s0, v14
; GFX12-NEXT: s_wait_alu depctr_va_vcc(0)
; GFX12-NEXT: v_add_co_ci_u32_e64 v15, null, s1, v15, vcc_lo
-; GFX12-NEXT: s_clause 0x2
+; GFX12-NEXT: s_clause 0x3
+; GFX12-NEXT: global_load_b128 v[2:5], v[8:9], off
; GFX12-NEXT: global_load_b128 v[6:9], v[6:7], off
; GFX12-NEXT: global_load_b128 v[10:13], v[10:11], off
; GFX12-NEXT: global_load_b128 v[14:17], v[14:15], off
+; GFX12-NEXT: s_wait_loadcnt 0x3
+; GFX12-NEXT: s_wait_kmcnt 0x0
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_add_nc_u32_e32 v5, s7, v5
+; GFX12-NEXT: v_add_nc_u32_e32 v4, s6, v4
+; GFX12-NEXT: v_add_nc_u32_e32 v3, s5, v3
+; GFX12-NEXT: v_add_nc_u32_e32 v2, s4, v2
; GFX12-NEXT: s_wait_loadcnt 0x2
; GFX12-NEXT: v_add_nc_u32_e32 v9, s7, v9
; GFX12-NEXT: v_add_nc_u32_e32 v8, s6, v8
>From 3e20adb56f180513becfbcd5fa32a5adf2b593cb Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Fri, 5 Jun 2026 16:45:36 -0500
Subject: [PATCH 6/7] Rephrase function comment.
---
llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
index a89805bb9595c..15aceafbab99d 100644
--- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
@@ -264,8 +264,8 @@ static bool shouldScheduleVOPDAdjacent(const TargetInstrInfo &TII,
.has_value();
}
-/// Check whether \p SU has a load dependency that is currently scheduled after
-/// \p Base (according to node number).
+/// Check whether \p SU has a load dependency whose node number comes after that
+/// of \p Base in the node vector.
static bool hasIntermediateLoadDependency(const SUnit &SU, const SUnit &Base) {
for (const SDep &Dep : SU.Preds) {
if (Dep.getKind() != SDep::Data)
>From f72c4ee9628634afb429a5def60fc700d9be5db6 Mon Sep 17 00:00:00 2001
From: Zach Goldthorpe <Zach.Goldthorpe at amd.com>
Date: Mon, 8 Jun 2026 16:48:55 -0500
Subject: [PATCH 7/7] Work-in-progress implementation of transitive load
checking
---
llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp | 57 +++-
.../AMDGPU/GlobalISel/insertelement.i8.ll | 15 +-
.../AMDGPU/GlobalISel/load-unaligned.ll | 10 +-
.../AMDGPU/GlobalISel/load-uniform-in-vgpr.ll | 88 +++---
llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll | 5 +-
.../CodeGen/AMDGPU/dagcombine-fma-fmad.ll | 20 +-
llvm/test/CodeGen/AMDGPU/fabs.bf16.ll | 89 +++---
llvm/test/CodeGen/AMDGPU/fneg.bf16.ll | 16 +-
llvm/test/CodeGen/AMDGPU/frem.ll | 9 +-
.../AMDGPU/gfx-callable-argument-types.ll | 6 +-
llvm/test/CodeGen/AMDGPU/idot4u.ll | 51 +--
.../llvm.amdgcn.struct.buffer.load.format.ll | 4 +-
llvm/test/CodeGen/AMDGPU/load-constant-i1.ll | 291 ++++++++++--------
llvm/test/CodeGen/AMDGPU/madak.ll | 6 +-
llvm/test/CodeGen/AMDGPU/scratch-simple.ll | 91 +++---
.../CodeGen/AMDGPU/vector_shuffle.packed.ll | 168 +++++-----
16 files changed, 488 insertions(+), 438 deletions(-)
diff --git a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
index 96ee47bcfdd13..e4380fb8e5d66 100644
--- a/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
+++ b/llvm/lib/Target/AMDGPU/GCNVOPDUtils.cpp
@@ -264,21 +264,52 @@ static bool shouldScheduleVOPDAdjacent(const TargetInstrInfo &TII,
.has_value();
}
-/// Check whether \p SU has a load dependency whose node number comes after that
-/// of \p Base in the node vector.
-static bool hasIntermediateLoadDependency(const SUnit &SU, const SUnit &Base) {
- for (const SDep &Dep : SU.Preds) {
- if (Dep.getKind() != SDep::Data)
+/// Collect all load (dependents if \p Forward else dependencies) that connect
+/// to \p SU.
+static void collectLoads(SmallVector<SUnit *> &Loads, BitVector &Seen,
+ const SUnit &SU, bool Forward) {
+ if (SU.isBoundaryNode() || Seen.test(SU.NodeNum))
+ return;
+
+ const SmallVector<SDep, 4> &Deps = Forward ? SU.Succs : SU.Preds;
+ for (const SDep &Edge : Deps) {
+ if (Edge.getKind() != SDep::Data)
continue;
- const SUnit *Pred = Dep.getSUnit();
- if (Pred->NodeNum < Base.NodeNum)
+ SUnit *Dep = Edge.getSUnit();
+ if (Dep->isBoundaryNode())
continue;
- if (!Pred->isInstr())
- continue;
- const MachineInstr *MI = Pred->getInstr();
- if (MI->mayLoad())
- return true;
+
+ if (Dep->isInstr() && Dep->getInstr()->mayLoad()) {
+ Loads.push_back(Dep);
+ } else {
+ collectLoads(Loads, Seen, *Dep, Forward);
+ }
+
+ Seen.set(Dep->NodeNum);
}
+}
+
+/// Check whether chaining \p First and \p Second together would force a load
+/// dependency of \p Second to complete before dispatching a load which depends
+/// on \p First when the loads would otherwise be overlapped. As this is a
+/// difficult condition to determine with certainty, it is made using the
+/// post-RA schedule.
+static bool fusionWouldSerializeOverlappingLoads(ScheduleDAGInstrs *DAG,
+ const SUnit &First,
+ const SUnit &Second) {
+ assert((First.isBoundaryNode() || First.NodeNum < Second.NodeNum) &&
+ "SUnit operands must be in topological order!");
+
+ BitVector Seen(DAG->SUnits.size());
+ SmallVector<SUnit *> FirstLoadSuccs, SecondLoadPreds;
+ collectLoads(FirstLoadSuccs, Seen, First, /*Forward=*/true);
+ collectLoads(SecondLoadPreds, Seen, Second, /*Forward=*/false);
+
+ for (SUnit *Succ : FirstLoadSuccs)
+ for (SUnit *Pred : SecondLoadPreds)
+ if (!DAG->IsReachable(Succ, Pred))
+ return true;
+
return false;
}
@@ -323,7 +354,7 @@ struct VOPDPairingMutation : ScheduleDAGMutation {
for (auto JSUI = ISUI + 1; JSUI != E; ++JSUI, ++JIdx) {
if (!VOPDCapable[JIdx] || JSUI->isBoundaryNode())
continue;
- if (hasIntermediateLoadDependency(*JSUI, *ISUI))
+ if (fusionWouldSerializeOverlappingLoads(DAG, *ISUI, *JSUI))
continue;
const MachineInstr *JMI = JSUI->getInstr();
if (!hasLessThanNumFused(*JSUI, 2) ||
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
index 6fda4137d6fea..439802c8b5731 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/insertelement.i8.ll
@@ -503,11 +503,10 @@ define amdgpu_ps void @insertelement_s_v2i8_s_v(ptr addrspace(4) inreg %ptr, i8
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xff, v1
; GFX11-FAKE16-NEXT: v_cndmask_b32_e64 v0, v2, s4, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xff, v0
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_lshlrev_b16 v3, 8, v1
-; GFX11-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v1, 0 :: v_dual_and_b32 v2, 0xff, v0
+; GFX11-FAKE16-NEXT: v_mov_b32_e32 v0, 0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_or_b32_e32 v2, v2, v3
; GFX11-FAKE16-NEXT: global_store_b16 v[0:1], v2, off
@@ -618,13 +617,13 @@ define amdgpu_ps void @insertelement_s_v2i8_v_v(ptr addrspace(4) inreg %ptr, i8
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v2, s1, v0, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_eq_u32_e32 vcc_lo, 0, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, s0, v0, vcc_lo
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
-; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v2.l
; GFX11-TRUE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-TRUE16-NEXT: v_and_b16 v0.h, 0xff, v1.l
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_lshlrev_b16 v0.l, 8, v0.l
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mov_b32_e32 v1, 0
; GFX11-TRUE16-NEXT: v_or_b16 v0.l, v0.h, v0.l
; GFX11-TRUE16-NEXT: global_store_b16 v[1:2], v0, off
; GFX11-TRUE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll
index b1de0eff05d30..25a373b1573c9 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-unaligned.ll
@@ -381,11 +381,11 @@ define amdgpu_ps void @test_s_load_constant_v8i32_align1(ptr addrspace(4) inreg
; GFX11-NEXT: v_readfirstlane_b32 s9, v5
; GFX11-NEXT: v_readfirstlane_b32 s10, v6
; GFX11-NEXT: v_readfirstlane_b32 s11, v7
-; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, s7
-; GFX11-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v2, s6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX11-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v7, s11
-; GFX11-NEXT: v_dual_mov_b32 v5, s9 :: v_dual_mov_b32 v6, s10
+; GFX11-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v1, s5
+; GFX11-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s7
+; GFX11-NEXT: v_dual_mov_b32 v4, s8 :: v_dual_mov_b32 v5, s9
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v7, s11
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b128 v8, v[0:3], s[2:3]
; GFX11-NEXT: global_store_b128 v8, v[4:7], s[2:3] offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
index a6d1393feca66..d78ccdc7527dd 100644
--- a/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
+++ b/llvm/test/CodeGen/AMDGPU/GlobalISel/load-uniform-in-vgpr.ll
@@ -425,12 +425,12 @@ define amdgpu_ps void @load_uniform_P1_v2i32(ptr addrspace(1) inreg %ptra, ptr a
; GFX11-NEXT: global_load_b64 v[2:3], v4, s[0:1]
; GFX11-NEXT: global_load_b64 v[4:5], v4, s[0:1] glc dlc
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_readfirstlane_b32 s1, v3
-; GFX11-NEXT: v_readfirstlane_b32 s3, v5
; GFX11-NEXT: v_readfirstlane_b32 s0, v2
+; GFX11-NEXT: v_readfirstlane_b32 s1, v3
; GFX11-NEXT: v_readfirstlane_b32 s2, v4
-; GFX11-NEXT: s_add_i32 s1, s1, s3
+; GFX11-NEXT: v_readfirstlane_b32 s3, v5
; GFX11-NEXT: s_add_i32 s0, s0, s2
+; GFX11-NEXT: s_add_i32 s1, s1, s3
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
@@ -443,12 +443,12 @@ define amdgpu_ps void @load_uniform_P1_v2i32(ptr addrspace(1) inreg %ptra, ptr a
; GFX12-NEXT: global_load_b64 v[2:3], v4, s[0:1]
; GFX12-NEXT: global_load_b64 v[4:5], v4, s[0:1] scope:SCOPE_SYS
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s1, v3
-; GFX12-NEXT: v_readfirstlane_b32 s3, v5
; GFX12-NEXT: v_readfirstlane_b32 s0, v2
+; GFX12-NEXT: v_readfirstlane_b32 s1, v3
; GFX12-NEXT: v_readfirstlane_b32 s2, v4
-; GFX12-NEXT: s_add_co_i32 s1, s1, s3
+; GFX12-NEXT: v_readfirstlane_b32 s3, v5
; GFX12-NEXT: s_add_co_i32 s0, s0, s2
+; GFX12-NEXT: s_add_co_i32 s1, s1, s3
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off
@@ -912,22 +912,21 @@ define amdgpu_ps void @load_uniform_P1_v16i32(ptr addrspace(1) inreg %ptra, ptr
; GFX11-NEXT: s_add_i32 s5, s5, s21
; GFX11-NEXT: s_add_i32 s6, s6, s22
; GFX11-NEXT: s_add_i32 s11, s11, s27
-; GFX11-NEXT: v_mov_b32_e32 v5, s3
+; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
; GFX11-NEXT: s_add_i32 s8, s8, s24
; GFX11-NEXT: s_add_i32 s9, s9, s25
; GFX11-NEXT: s_add_i32 s10, s10, s26
; GFX11-NEXT: s_add_i32 s15, s15, s31
-; GFX11-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v9, s7
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
; GFX11-NEXT: s_add_i32 s12, s12, s28
; GFX11-NEXT: s_add_i32 s13, s13, s29
; GFX11-NEXT: s_add_i32 s14, s14, s30
-; GFX11-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v7, s5
-; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v13, s11
-; GFX11-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v11, s9
-; GFX11-NEXT: v_dual_mov_b32 v10, s8 :: v_dual_mov_b32 v17, s15
-; GFX11-NEXT: v_dual_mov_b32 v16, s14 :: v_dual_mov_b32 v15, s13
-; GFX11-NEXT: v_mov_b32_e32 v14, s12
+; GFX11-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
+; GFX11-NEXT: v_dual_mov_b32 v13, s11 :: v_dual_mov_b32 v12, s10
+; GFX11-NEXT: v_dual_mov_b32 v11, s9 :: v_dual_mov_b32 v10, s8
+; GFX11-NEXT: v_dual_mov_b32 v17, s15 :: v_dual_mov_b32 v16, s14
+; GFX11-NEXT: v_dual_mov_b32 v15, s13 :: v_dual_mov_b32 v14, s12
; GFX11-NEXT: s_clause 0x3
; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off
; GFX11-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
@@ -992,22 +991,21 @@ define amdgpu_ps void @load_uniform_P1_v16i32(ptr addrspace(1) inreg %ptra, ptr
; GFX12-NEXT: s_add_co_i32 s5, s5, s21
; GFX12-NEXT: s_add_co_i32 s6, s6, s22
; GFX12-NEXT: s_add_co_i32 s11, s11, s27
-; GFX12-NEXT: v_mov_b32_e32 v5, s3
+; GFX12-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
; GFX12-NEXT: s_add_co_i32 s8, s8, s24
; GFX12-NEXT: s_add_co_i32 s9, s9, s25
; GFX12-NEXT: s_add_co_i32 s10, s10, s26
; GFX12-NEXT: s_add_co_i32 s15, s15, s31
-; GFX12-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v3, s1
-; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v9, s7
+; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX12-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
; GFX12-NEXT: s_add_co_i32 s12, s12, s28
; GFX12-NEXT: s_add_co_i32 s13, s13, s29
; GFX12-NEXT: s_add_co_i32 s14, s14, s30
-; GFX12-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v7, s5
-; GFX12-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v13, s11
-; GFX12-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v11, s9
-; GFX12-NEXT: v_dual_mov_b32 v10, s8 :: v_dual_mov_b32 v17, s15
-; GFX12-NEXT: v_dual_mov_b32 v16, s14 :: v_dual_mov_b32 v15, s13
-; GFX12-NEXT: v_mov_b32_e32 v14, s12
+; GFX12-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
+; GFX12-NEXT: v_dual_mov_b32 v13, s11 :: v_dual_mov_b32 v12, s10
+; GFX12-NEXT: v_dual_mov_b32 v11, s9 :: v_dual_mov_b32 v10, s8
+; GFX12-NEXT: v_dual_mov_b32 v17, s15 :: v_dual_mov_b32 v16, s14
+; GFX12-NEXT: v_dual_mov_b32 v15, s13 :: v_dual_mov_b32 v14, s12
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v[0:1], v[2:5], off
; GFX12-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
@@ -1632,11 +1630,11 @@ define amdgpu_ps void @load_uniform_P4_v2i32(ptr addrspace(4) inreg %ptra, ptr a
; GFX11-NEXT: global_load_b64 v[2:3], v2, s[0:1]
; GFX11-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_readfirstlane_b32 s3, v3
; GFX11-NEXT: v_readfirstlane_b32 s2, v2
+; GFX11-NEXT: v_readfirstlane_b32 s3, v3
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: s_add_i32 s1, s3, s1
; GFX11-NEXT: s_add_i32 s0, s2, s0
+; GFX11-NEXT: s_add_i32 s1, s3, s1
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX11-NEXT: global_store_b64 v[0:1], v[2:3], off
@@ -1648,11 +1646,11 @@ define amdgpu_ps void @load_uniform_P4_v2i32(ptr addrspace(4) inreg %ptra, ptr a
; GFX12-NEXT: global_load_b64 v[2:3], v2, s[0:1]
; GFX12-NEXT: s_load_b64 s[0:1], s[0:1], 0x0
; GFX12-NEXT: s_wait_loadcnt 0x0
-; GFX12-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-NEXT: v_readfirstlane_b32 s2, v2
+; GFX12-NEXT: v_readfirstlane_b32 s3, v3
; GFX12-NEXT: s_wait_kmcnt 0x0
-; GFX12-NEXT: s_add_co_i32 s1, s3, s1
; GFX12-NEXT: s_add_co_i32 s0, s2, s0
+; GFX12-NEXT: s_add_co_i32 s1, s3, s1
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX12-NEXT: global_store_b64 v[0:1], v[2:3], off
@@ -2036,22 +2034,21 @@ define amdgpu_ps void @load_uniform_P4_v16i32(ptr addrspace(4) inreg %ptra, ptr
; GFX11-NEXT: s_add_i32 s5, s21, s5
; GFX11-NEXT: s_add_i32 s6, s22, s6
; GFX11-NEXT: s_add_i32 s11, s27, s11
-; GFX11-NEXT: v_mov_b32_e32 v5, s3
+; GFX11-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
; GFX11-NEXT: s_add_i32 s8, s24, s8
; GFX11-NEXT: s_add_i32 s9, s25, s9
; GFX11-NEXT: s_add_i32 s10, s26, s10
; GFX11-NEXT: s_add_i32 s15, s31, s15
-; GFX11-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v3, s1
-; GFX11-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v9, s7
+; GFX11-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX11-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
; GFX11-NEXT: s_add_i32 s12, s28, s12
; GFX11-NEXT: s_add_i32 s13, s29, s13
; GFX11-NEXT: s_add_i32 s14, s30, s14
-; GFX11-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v7, s5
-; GFX11-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v13, s11
-; GFX11-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v11, s9
-; GFX11-NEXT: v_dual_mov_b32 v10, s8 :: v_dual_mov_b32 v17, s15
-; GFX11-NEXT: v_dual_mov_b32 v16, s14 :: v_dual_mov_b32 v15, s13
-; GFX11-NEXT: v_mov_b32_e32 v14, s12
+; GFX11-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
+; GFX11-NEXT: v_dual_mov_b32 v13, s11 :: v_dual_mov_b32 v12, s10
+; GFX11-NEXT: v_dual_mov_b32 v11, s9 :: v_dual_mov_b32 v10, s8
+; GFX11-NEXT: v_dual_mov_b32 v17, s15 :: v_dual_mov_b32 v16, s14
+; GFX11-NEXT: v_dual_mov_b32 v15, s13 :: v_dual_mov_b32 v14, s12
; GFX11-NEXT: s_clause 0x3
; GFX11-NEXT: global_store_b128 v[0:1], v[2:5], off
; GFX11-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
@@ -2098,22 +2095,21 @@ define amdgpu_ps void @load_uniform_P4_v16i32(ptr addrspace(4) inreg %ptra, ptr
; GFX12-NEXT: s_add_co_i32 s5, s21, s5
; GFX12-NEXT: s_add_co_i32 s6, s22, s6
; GFX12-NEXT: s_add_co_i32 s11, s27, s11
-; GFX12-NEXT: v_mov_b32_e32 v5, s3
+; GFX12-NEXT: v_dual_mov_b32 v5, s3 :: v_dual_mov_b32 v4, s2
; GFX12-NEXT: s_add_co_i32 s8, s24, s8
; GFX12-NEXT: s_add_co_i32 s9, s25, s9
; GFX12-NEXT: s_add_co_i32 s10, s26, s10
; GFX12-NEXT: s_add_co_i32 s15, s31, s15
-; GFX12-NEXT: v_dual_mov_b32 v4, s2 :: v_dual_mov_b32 v3, s1
-; GFX12-NEXT: v_dual_mov_b32 v2, s0 :: v_dual_mov_b32 v9, s7
+; GFX12-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
+; GFX12-NEXT: v_dual_mov_b32 v9, s7 :: v_dual_mov_b32 v8, s6
; GFX12-NEXT: s_add_co_i32 s12, s28, s12
; GFX12-NEXT: s_add_co_i32 s13, s29, s13
; GFX12-NEXT: s_add_co_i32 s14, s30, s14
-; GFX12-NEXT: v_dual_mov_b32 v8, s6 :: v_dual_mov_b32 v7, s5
-; GFX12-NEXT: v_dual_mov_b32 v6, s4 :: v_dual_mov_b32 v13, s11
-; GFX12-NEXT: v_dual_mov_b32 v12, s10 :: v_dual_mov_b32 v11, s9
-; GFX12-NEXT: v_dual_mov_b32 v10, s8 :: v_dual_mov_b32 v17, s15
-; GFX12-NEXT: v_dual_mov_b32 v16, s14 :: v_dual_mov_b32 v15, s13
-; GFX12-NEXT: v_mov_b32_e32 v14, s12
+; GFX12-NEXT: v_dual_mov_b32 v7, s5 :: v_dual_mov_b32 v6, s4
+; GFX12-NEXT: v_dual_mov_b32 v13, s11 :: v_dual_mov_b32 v12, s10
+; GFX12-NEXT: v_dual_mov_b32 v11, s9 :: v_dual_mov_b32 v10, s8
+; GFX12-NEXT: v_dual_mov_b32 v17, s15 :: v_dual_mov_b32 v16, s14
+; GFX12-NEXT: v_dual_mov_b32 v15, s13 :: v_dual_mov_b32 v14, s12
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v[0:1], v[2:5], off
; GFX12-NEXT: global_store_b128 v[0:1], v[6:9], off offset:16
diff --git a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
index 0eed0ba50092b..bb9595d8592c0 100644
--- a/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
+++ b/llvm/test/CodeGen/AMDGPU/cvt_f32_ubyte.ll
@@ -1551,8 +1551,7 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned_multiuse(ptr addrspace(1
; GFX11: ; %bb.0:
; GFX11-NEXT: s_load_b256 s[0:7], s[4:5], 0x24
; GFX11-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-NEXT: v_mov_b32_e32 v6, 0
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-NEXT: s_clause 0x3
@@ -1565,10 +1564,10 @@ define amdgpu_kernel void @load_v4i8_to_v4f32_unaligned_multiuse(ptr addrspace(1
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v1, v1
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_lshl_or_b32 v5, v2, 8, v0
+; GFX11-NEXT: v_mov_b32_e32 v6, 0
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v2, v0
; GFX11-NEXT: v_cvt_f32_ubyte0_e32 v0, v3
; GFX11-NEXT: v_mov_b32_e32 v3, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-NEXT: v_perm_b32 v4, v4, v5, 0x4000405
; GFX11-NEXT: s_clause 0x1
; GFX11-NEXT: global_store_b128 v6, v[0:3], s[0:1]
diff --git a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
index d27ce714d848b..3a158beb8ba91 100644
--- a/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
+++ b/llvm/test/CodeGen/AMDGPU/dagcombine-fma-fmad.ll
@@ -109,35 +109,35 @@ define amdgpu_ps float @_amdgpu_ps_main() #0 {
; GFX11-NEXT: v_sub_f32_e32 v8, s0, v1
; GFX11-NEXT: v_fma_f32 v7, -s2, v6, s6
; GFX11-NEXT: v_fma_f32 v10, s2, v6, v2
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_fma_f32 v5, v6, v5, 1.0
; GFX11-NEXT: s_mov_b32 s0, 0x3c23d70a
; GFX11-NEXT: s_waitcnt lgkmcnt(0)
-; GFX11-NEXT: v_mul_f32_e32 v9, s10, v0
+; GFX11-NEXT: v_dual_fmac_f32 v10, v7, v6 :: v_dual_mul_f32 v9, s10, v0
; GFX11-NEXT: v_fma_f32 v0, -v0, s10, s14
; GFX11-NEXT: v_dual_fmac_f32 v1, v6, v8 :: v_dual_mul_f32 v8, s18, v2
-; GFX11-NEXT: v_dual_mul_f32 v3, s22, v3 :: v_dual_fmac_f32 v10, v7, v6
+; GFX11-NEXT: v_mul_f32_e32 v3, s22, v3
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_dual_fmac_f32 v9, v0, v6 :: v_dual_sub_f32 v0, v1, v5
; GFX11-NEXT: v_mul_f32_e32 v7, v6, v3
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_fma_f32 v3, -v6, v3, v9
+; GFX11-NEXT: s_waitcnt vmcnt(0)
+; GFX11-NEXT: v_add_f32_e32 v4, v4, v10
; GFX11-NEXT: v_fmac_f32_e32 v7, v3, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fmac_f32_e32 v5, v0, v6
; GFX11-NEXT: v_mul_f32_e32 v1, v8, v6
-; GFX11-NEXT: s_waitcnt vmcnt(0)
-; GFX11-NEXT: v_add_f32_e32 v4, v4, v10
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-NEXT: v_dual_mul_f32 v3, v4, v6 :: v_dual_fmaak_f32 v4, s0, v5, 0x3ca3d70a
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fma_f32 v0, v2, s26, -v1
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-NEXT: v_fmac_f32_e32 v1, v0, v6
; GFX11-NEXT: v_mul_f32_e32 v0, v2, v6
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-NEXT: v_mul_f32_e32 v2, v7, v4
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_mul_f32_e32 v1, v3, v1
-; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-NEXT: v_fmac_f32_e32 v1, v2, v0
+; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_max_f32_e32 v0, 0, v1
; GFX11-NEXT: ; return to shader part epilog
.entry:
diff --git a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
index 9018d71e7a59c..8717e20ed0184 100644
--- a/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fabs.bf16.ll
@@ -624,20 +624,20 @@ define amdgpu_kernel void @v_fabs_fold_self_v2bf16(ptr addrspace(1) %out, ptr ad
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v2.h
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v3, 16, v2
; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_dual_mul_f32 v0, v0, v3 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_mul_f32 v1, v1, v2 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_mul_f32_e32 v1, v1, v2
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, v0, v3
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_cndmask_b32 v1, v3, v5
@@ -806,11 +806,13 @@ define amdgpu_kernel void @v_fabs_fold_v2bf16(ptr addrspace(1) %out, ptr addrspa
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_dual_mul_f32 v0, s2, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX11-TRUE16-NEXT: s_and_b32 s2, s4, 0xffff0000
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instid1(SALU_CYCLE_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instid1(SALU_CYCLE_1)
; GFX11-TRUE16-NEXT: v_mul_f32_e32 v1, s2, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
@@ -842,25 +844,27 @@ define amdgpu_kernel void @v_fabs_fold_v2bf16(ptr addrspace(1) %out, ptr addrspa
; GFX11-FAKE16-NEXT: s_and_b32 s3, s4, 0xffff0000
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fff, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0x7fff, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_mul_f32 v1, s3, v1 :: v_dual_and_b32 v0, 0x7fff, v0
+; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_dual_mul_f32 v0, s2, v0 :: v_dual_and_b32 v1, 0x7fff, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_mul_f32_e32 v0, s2, v0
; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-FAKE16-NEXT: v_dual_mul_f32 v1, s3, v1 :: v_dual_mov_b32 v2, 0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
+; GFX11-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_cndmask_b32 v1, v3, v5
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_perm_b32 v0, v1, v0, 0x7060302
; GFX11-FAKE16-NEXT: global_store_b32 v2, v0, s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
@@ -979,20 +983,20 @@ define amdgpu_kernel void @v_extract_fabs_fold_v2bf16(ptr addrspace(1) %in) #0 {
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b16 v0.l, 0x7fff, v0.l
; GFX11-TRUE16-NEXT: v_and_b16 v1.l, 0x7fff, v0.h
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-TRUE16-NEXT: v_dual_mul_f32 v0, 4.0, v0 :: v_dual_lshlrev_b32 v1, 16, v1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-TRUE16-NEXT: v_dual_add_f32 v1, 2.0, v1 :: v_dual_lshlrev_b32 v0, 16, v0
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_add_f32_e32 v1, 2.0, v1
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX11-TRUE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
@@ -1006,30 +1010,31 @@ define amdgpu_kernel void @v_extract_fabs_fold_v2bf16(ptr addrspace(1) %in) #0 {
; GFX11-FAKE16: ; %bb.0:
; GFX11-FAKE16-NEXT: s_load_b64 s[0:1], s[4:5], 0x0
; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-FAKE16-NEXT: global_load_b32 v0, v0, s[0:1]
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0x7fff, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0x7fff, v1
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
+; GFX11-FAKE16-NEXT: v_dual_add_f32 v1, 2.0, v1 :: v_dual_and_b32 v0, 0x7fff, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
+; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
-; GFX11-FAKE16-NEXT: v_dual_mul_f32 v0, 4.0, v0 :: v_dual_and_b32 v1, 0x7fff, v1
+; GFX11-FAKE16-NEXT: v_mul_f32_e32 v0, 4.0, v0
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v4, 0x400000, v0
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_cndmask_b32 v0, v2, v4 :: v_dual_lshlrev_b32 v1, 16, v1
-; GFX11-FAKE16-NEXT: v_add_f32_e32 v1, 2.0, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v3, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v5, 0x400000, v1
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-FAKE16-NEXT: v_add3_u32 v3, v3, v1, 0x7fff
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v4, vcc_lo
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v3, v5, vcc_lo
; GFX11-FAKE16-NEXT: global_store_d16_hi_b16 v[0:1], v0, off dlc
; GFX11-FAKE16-NEXT: s_waitcnt_vscnt null, 0x0
diff --git a/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll b/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
index f2f97faafdd1d..63aadaacbeb3a 100644
--- a/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
+++ b/llvm/test/CodeGen/AMDGPU/fneg.bf16.ll
@@ -311,13 +311,13 @@ define amdgpu_kernel void @v_fneg_fold_bf16(ptr addrspace(1) %out, ptr addrspace
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-TRUE16-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, v2, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc_lo
; GFX11-TRUE16-NEXT: global_store_d16_hi_b16 v0, v1, s[0:1]
; GFX11-TRUE16-NEXT: s_endpgm
@@ -334,13 +334,13 @@ define amdgpu_kernel void @v_fneg_fold_bf16(ptr addrspace(1) %out, ptr addrspace
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v2, 16, v2
; GFX11-FAKE16-NEXT: v_mul_f32_e32 v1, v2, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, v2, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_2) | instid1(VALU_DEP_2)
; GFX11-FAKE16-NEXT: v_add_nc_u32_e32 v2, 0x7fff, v2
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v1
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v2, v3, vcc_lo
; GFX11-FAKE16-NEXT: global_store_d16_hi_b16 v0, v1, s[0:1]
; GFX11-FAKE16-NEXT: s_endpgm
diff --git a/llvm/test/CodeGen/AMDGPU/frem.ll b/llvm/test/CodeGen/AMDGPU/frem.ll
index 4a0028a3e733e..3ba9d5ff7369f 100644
--- a/llvm/test/CodeGen/AMDGPU/frem.ll
+++ b/llvm/test/CodeGen/AMDGPU/frem.ll
@@ -17711,11 +17711,12 @@ define amdgpu_kernel void @frem_v2f64_const_zero_num(ptr addrspace(1) %out, ptr
; GFX11-NEXT: global_load_b128 v[1:4], v0, s[2:3]
; GFX11-NEXT: s_waitcnt vmcnt(0)
; GFX11-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[1:2]
+; GFX11-NEXT: v_mov_b32_e32 v2, v0
; GFX11-NEXT: s_and_b32 s2, vcc_lo, exec_lo
; GFX11-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[3:4]
; GFX11-NEXT: s_cselect_b32 s2, 0x7ff80000, 0
; GFX11-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX11-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, v0
+; GFX11-NEXT: v_mov_b32_e32 v1, s2
; GFX11-NEXT: s_and_b32 s3, vcc_lo, exec_lo
; GFX11-NEXT: s_cselect_b32 s3, 0x7ff80000, 0
; GFX11-NEXT: v_mov_b32_e32 v3, s3
@@ -17730,11 +17731,12 @@ define amdgpu_kernel void @frem_v2f64_const_zero_num(ptr addrspace(1) %out, ptr
; GFX1150-NEXT: global_load_b128 v[1:4], v0, s[2:3]
; GFX1150-NEXT: s_waitcnt vmcnt(0)
; GFX1150-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[1:2]
+; GFX1150-NEXT: v_mov_b32_e32 v2, v0
; GFX1150-NEXT: s_and_b32 s2, vcc_lo, exec_lo
; GFX1150-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[3:4]
; GFX1150-NEXT: s_cselect_b32 s2, 0x7ff80000, 0
; GFX1150-NEXT: s_delay_alu instid0(SALU_CYCLE_1) | instskip(SKIP_2) | instid1(SALU_CYCLE_1)
-; GFX1150-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, v0
+; GFX1150-NEXT: v_mov_b32_e32 v1, s2
; GFX1150-NEXT: s_and_b32 s3, vcc_lo, exec_lo
; GFX1150-NEXT: s_cselect_b32 s3, 0x7ff80000, 0
; GFX1150-NEXT: v_mov_b32_e32 v3, s3
@@ -17749,11 +17751,12 @@ define amdgpu_kernel void @frem_v2f64_const_zero_num(ptr addrspace(1) %out, ptr
; GFX1200-NEXT: global_load_b128 v[1:4], v0, s[2:3]
; GFX1200-NEXT: s_wait_loadcnt 0x0
; GFX1200-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[1:2]
+; GFX1200-NEXT: v_mov_b32_e32 v2, v0
; GFX1200-NEXT: s_and_b32 s2, vcc_lo, exec_lo
; GFX1200-NEXT: v_cmp_nlg_f64_e32 vcc_lo, 0, v[3:4]
; GFX1200-NEXT: s_cselect_b32 s2, 0x7ff80000, 0
; GFX1200-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1200-NEXT: v_dual_mov_b32 v1, s2 :: v_dual_mov_b32 v2, v0
+; GFX1200-NEXT: v_mov_b32_e32 v1, s2
; GFX1200-NEXT: s_and_b32 s3, vcc_lo, exec_lo
; GFX1200-NEXT: s_cselect_b32 s3, 0x7ff80000, 0
; GFX1200-NEXT: s_wait_alu depctr_sa_sdst(0)
diff --git a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
index 100707a5041d7..6922dc03f3203 100644
--- a/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
+++ b/llvm/test/CodeGen/AMDGPU/gfx-callable-argument-types.ll
@@ -9468,10 +9468,10 @@ define amdgpu_gfx void @test_call_external_void_func_v16i8() #0 {
; GFX11-NEXT: v_lshrrev_b32_e32 v13, 8, v3
; GFX11-NEXT: v_lshrrev_b32_e32 v14, 16, v3
; GFX11-NEXT: v_lshrrev_b32_e32 v15, 24, v3
-; GFX11-NEXT: v_mov_b32_e32 v4, v1
+; GFX11-NEXT: v_dual_mov_b32 v4, v1 :: v_dual_mov_b32 v1, v16
; GFX11-NEXT: v_mov_b32_e32 v8, v2
-; GFX11-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v1, v16
-; GFX11-NEXT: v_dual_mov_b32 v2, v17 :: v_dual_mov_b32 v3, v18
+; GFX11-NEXT: v_dual_mov_b32 v12, v3 :: v_dual_mov_b32 v3, v18
+; GFX11-NEXT: v_mov_b32_e32 v2, v17
; GFX11-NEXT: s_swappc_b64 s[30:31], s[0:1]
; GFX11-NEXT: v_readlane_b32 s30, v40, 0
; GFX11-NEXT: v_readlane_b32 s31, v40, 1
diff --git a/llvm/test/CodeGen/AMDGPU/idot4u.ll b/llvm/test/CodeGen/AMDGPU/idot4u.ll
index b67c40b2d8333..1735e5c8c4ca2 100644
--- a/llvm/test/CodeGen/AMDGPU/idot4u.ll
+++ b/llvm/test/CodeGen/AMDGPU/idot4u.ll
@@ -1682,30 +1682,30 @@ define amdgpu_kernel void @notdot4_mixedtypes(ptr addrspace(1) %src1,
; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DL-FAKE16-NEXT: s_clause 0x1
; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]
; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]
-; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)
+; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v2, s[4:5]
+; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(2)
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
-; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v6, v1, 0, 8
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v7, v0, 0, 8
+; GFX11-DL-FAKE16-NEXT: v_perm_b32 v0, v0, v0, 0xc0c0302
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v4
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v5
-; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v2, s[4:5]
-; GFX11-DL-FAKE16-NEXT: v_perm_b32 v0, v0, v0, 0xc0c0302
; GFX11-DL-FAKE16-NEXT: v_perm_b32 v1, v1, v1, 0xc0c0302
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)
+; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v4, v5, v3
-; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v6, v7, v3
+; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff, v3
-; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_dot4_u32_u8 v0, v1, v0, v3
; GFX11-DL-FAKE16-NEXT: global_store_b16 v2, v0, s[4:5]
; GFX11-DL-FAKE16-NEXT: s_endpgm
@@ -1977,35 +1977,35 @@ define amdgpu_kernel void @notdot4_mixedtypes2(ptr addrspace(1) %src1,
; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DL-FAKE16-NEXT: s_clause 0x1
; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]
; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]
-; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
-; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
-; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v9, 0xff, v0
; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v2, s[4:5]
+; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v4, 8, v1
+; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 8, v0
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v6, 16, v1
-; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v4
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v7, 16, v0
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v8, v1, 0, 8
+; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v4, 0xff, v4
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v5, v5, 0, 8
+; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v9, 0xff, v0
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 24, v1
-; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_3)
-; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v4, v5, v3
; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v4, v6, 0, 8
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v5, 0xff, v7
+; GFX11-DL-FAKE16-NEXT: v_bfe_i32 v0, v0, 0, 8
+; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v8, v9, v3
-; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_mad_u16 v3, v4, v5, v3
+; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-DL-FAKE16-NEXT: v_mad_u16 v0, v1, v0, v3
; GFX11-DL-FAKE16-NEXT: global_store_b16 v2, v0, s[4:5]
; GFX11-DL-FAKE16-NEXT: s_endpgm
@@ -2451,24 +2451,25 @@ define amdgpu_kernel void @udot4_acc16_vecMul(ptr addrspace(1) %src1,
; GFX11-DL-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v0, 0x3ff, v0
; GFX11-DL-FAKE16-NEXT: s_load_b64 s[4:5], s[4:5], 0x34
-; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-DL-FAKE16-NEXT: v_mov_b32_e32 v2, 0
+; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX11-DL-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 2, v0
; GFX11-DL-FAKE16-NEXT: s_waitcnt lgkmcnt(0)
; GFX11-DL-FAKE16-NEXT: s_clause 0x1
; GFX11-DL-FAKE16-NEXT: global_load_b32 v1, v0, s[0:1]
; GFX11-DL-FAKE16-NEXT: global_load_b32 v0, v0, s[2:3]
-; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)
-; GFX11-DL-FAKE16-NEXT: v_dual_mov_b32 v2, 0 :: v_dual_and_b32 v7, 0xff, v1
; GFX11-DL-FAKE16-NEXT: global_load_u16 v3, v2, s[4:5]
+; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(2)
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v4, 8, v1
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(1)
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b16 v5, 8, v0
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v6, 0xff, v0
+; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v7, 0xff, v1
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v8, 16, v1
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v9, 16, v0
-; GFX11-DL-FAKE16-NEXT: v_perm_b32 v4, v4, v7, 0x5040100
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 24, v1
; GFX11-DL-FAKE16-NEXT: v_perm_b32 v5, v5, v6, 0x5040100
+; GFX11-DL-FAKE16-NEXT: v_perm_b32 v4, v4, v7, 0x5040100
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v0, 24, v0
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v6, 0xff, v9
; GFX11-DL-FAKE16-NEXT: v_and_b32_e32 v7, 0xff, v8
@@ -2478,12 +2479,12 @@ define amdgpu_kernel void @udot4_acc16_vecMul(ptr addrspace(1) %src1,
; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-DL-FAKE16-NEXT: v_perm_b32 v1, v1, v7, 0x5040100
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v5, 16, v4
-; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
-; GFX11-DL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v1, v0
; GFX11-DL-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v3, v4, v3
-; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v1, v3, v5
; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-DL-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v1, v0
+; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v1, v3, v5
+; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-DL-FAKE16-NEXT: v_lshrrev_b32_e32 v3, 16, v0
; GFX11-DL-FAKE16-NEXT: v_add_nc_u16 v0, v1, v0
; GFX11-DL-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
diff --git a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
index ab65840653c80..74e3bccd3418c 100644
--- a/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
+++ b/llvm/test/CodeGen/AMDGPU/llvm.amdgcn.struct.buffer.load.format.ll
@@ -1254,8 +1254,8 @@ define amdgpu_cs float @buffer_load_v2i32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
; GFX12-GISEL-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
; GFX12-GISEL-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v3
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v2
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v3
; GFX12-GISEL-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX12-GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v4
@@ -1337,8 +1337,8 @@ define amdgpu_cs float @buffer_load_v2f32_tfe(<4 x i32> inreg %rsrc, ptr addrspa
; GFX12-GISEL-NEXT: v_dual_mov_b32 v3, v2 :: v_dual_mov_b32 v4, v2
; GFX12-GISEL-NEXT: buffer_load_format_xy v[2:4], v2, s[0:3], null idxen tfe
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
-; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v3
; GFX12-GISEL-NEXT: v_readfirstlane_b32 s0, v2
+; GFX12-GISEL-NEXT: v_readfirstlane_b32 s1, v3
; GFX12-GISEL-NEXT: v_dual_mov_b32 v3, s1 :: v_dual_mov_b32 v2, s0
; GFX12-GISEL-NEXT: global_store_b64 v[0:1], v[2:3], off
; GFX12-GISEL-NEXT: v_mov_b32_e32 v0, v4
diff --git a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
index f2b5c7679dfe9..01bdaacbabac2 100644
--- a/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
+++ b/llvm/test/CodeGen/AMDGPU/load-constant-i1.ll
@@ -1708,19 +1708,19 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i32(ptr addrspace(1) %out
; GFX12-NEXT: global_load_d16_u8 v0, v8, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_readfirstlane_b32 s2, v0
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10001
+; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10005
; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v5, s4 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX12-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_and_b32 v0, 0xffff, v0
; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10003
-; GFX12-NEXT: s_bfe_u32 s5, s2, 0x10005
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x10001
; GFX12-NEXT: s_and_b32 s6, s2, 1
; GFX12-NEXT: s_bfe_u32 s7, s2, 0x10002
; GFX12-NEXT: s_bfe_u32 s2, s2, 0x10004
; GFX12-NEXT: v_lshrrev_b32_e32 v3, 7, v0
; GFX12-NEXT: v_bfe_u32 v2, v0, 6, 1
-; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v7, s3
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v1, s5 :: v_dual_mov_b32 v4, s6
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s4
+; GFX12-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s3
; GFX12-NEXT: v_mov_b32_e32 v6, s7
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
@@ -1772,9 +1772,9 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i32(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_lshr_b32 s9, s5, 7
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s5, 0x10006
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s6
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v4, s7 :: v_dual_mov_b32 v2, s5
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, s9 :: v_dual_mov_b32 v5, s4
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v7, s3
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v2, s5
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, s9 :: v_dual_mov_b32 v7, s3
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v4, s7 :: v_dual_mov_b32 v5, s4
; GFX1250-REAL16-NEXT: s_clause 0x1
; GFX1250-REAL16-NEXT: global_store_b128 v8, v[0:3], s[0:1] offset:16
; GFX1250-REAL16-NEXT: global_store_b128 v8, v[4:7], s[0:1]
@@ -2104,15 +2104,14 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX12-NEXT: s_bfe_u32 s16, s3, 0x10004
; GFX12-NEXT: s_bfe_u32 s17, s3, 0x10008
; GFX12-NEXT: s_bfe_u32 s3, s3, 0x1000e
-; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s7
-; GFX12-NEXT: v_mov_b32_e32 v1, s8
-; GFX12-NEXT: v_dual_mov_b32 v15, s4 :: v_dual_mov_b32 v2, s3
-; GFX12-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v4, s17
-; GFX12-NEXT: v_dual_mov_b32 v6, s10 :: v_dual_mov_b32 v11, s6
-; GFX12-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v8, s16
-; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s15
-; GFX12-NEXT: v_dual_mov_b32 v12, s9 :: v_dual_mov_b32 v13, s5
-; GFX12-NEXT: v_mov_b32_e32 v14, s14
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
+; GFX12-NEXT: v_dual_mov_b32 v13, s5 :: v_dual_mov_b32 v2, s3
+; GFX12-NEXT: v_dual_mov_b32 v3, s13 :: v_dual_mov_b32 v8, s16
+; GFX12-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s10
+; GFX12-NEXT: v_dual_mov_b32 v15, s4 :: v_dual_mov_b32 v4, s17
+; GFX12-NEXT: v_dual_mov_b32 v7, s12 :: v_dual_mov_b32 v10, s15
+; GFX12-NEXT: v_dual_mov_b32 v11, s6 :: v_dual_mov_b32 v12, s9
+; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v14, s14
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX12-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
@@ -2147,13 +2146,13 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i32(ptr addrspace(1) %o
; GFX1250-NEXT: s_bfe_u32 s17, s6, 0x10008
; GFX1250-NEXT: s_bfe_u32 s6, s6, 0x1000e
; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s8
-; GFX1250-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v15, s3
; GFX1250-NEXT: v_dual_mov_b32 v2, s6 :: v_dual_mov_b32 v3, s13
-; GFX1250-NEXT: v_dual_mov_b32 v4, s17 :: v_dual_mov_b32 v6, s10
-; GFX1250-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v7, s12
+; GFX1250-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s10
+; GFX1250-NEXT: v_dual_mov_b32 v4, s17 :: v_dual_mov_b32 v7, s12
+; GFX1250-NEXT: v_dual_mov_b32 v11, s5 :: v_dual_mov_b32 v12, s9
; GFX1250-NEXT: v_dual_mov_b32 v8, s16 :: v_dual_mov_b32 v9, s11
-; GFX1250-NEXT: v_dual_mov_b32 v10, s15 :: v_dual_mov_b32 v12, s9
-; GFX1250-NEXT: v_dual_mov_b32 v13, s4 :: v_dual_mov_b32 v14, s14
+; GFX1250-NEXT: v_dual_mov_b32 v10, s15 :: v_dual_mov_b32 v14, s14
+; GFX1250-NEXT: v_dual_mov_b32 v13, s4 :: v_dual_mov_b32 v15, s3
; GFX1250-NEXT: s_clause 0x3
; GFX1250-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX1250-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
@@ -5346,15 +5345,17 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_mov_b32_e32 v3, v1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_d16_u8 v0, v1, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v0
; GFX12-NEXT: v_and_b32_e32 v0, 1, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
-; GFX12-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_and_b32 v0, 0xffff, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-NEXT: v_lshrrev_b32_e32 v2, 1, v2
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX12-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX12-NEXT: s_endpgm
@@ -5364,16 +5365,16 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_4) | instid1(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-FAKE16-NEXT: global_load_u8 v0, v1, s[2:3] nv
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v0
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX1250-FAKE16-NEXT: v_dual_lshrrev_b32 v2, 1, v2 :: v_dual_bitop2_b32 v0, 1, v0 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, v1
-; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX1250-FAKE16-NEXT: s_endpgm
;
@@ -5382,6 +5383,8 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
@@ -5393,7 +5396,6 @@ define amdgpu_kernel void @constant_zextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
; GFX1250-REAL16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v2, s3
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-REAL16-NEXT: global_store_b128 v1, v[0:3], s[0:1]
; GFX1250-REAL16-NEXT: s_endpgm
%load = load <2 x i1>, ptr addrspace(4) %in
@@ -5491,9 +5493,11 @@ define amdgpu_kernel void @constant_sextload_v2i1_to_v2i64(ptr addrspace(1) %out
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_lshrrev_b32_e32 v2, 1, v0
; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX1250-NEXT: v_bfe_i32 v2, v2, 0, 1
-; GFX1250-NEXT: v_dual_ashrrev_i32 v1, 31, v0 :: v_dual_ashrrev_i32 v3, 31, v2
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX1250-NEXT: global_store_b128 v4, v[0:3], s[0:1]
; GFX1250-NEXT: s_endpgm
%load = load <2 x i1>, ptr addrspace(4) %in
@@ -5586,13 +5590,13 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_and_b32_e32 v1, 0xffff, v0
; GFX12-NEXT: v_bfe_u32 v2, v0, 1, 1
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX12-NEXT: v_dual_mov_b32 v3, v5 :: v_dual_and_b32 v0, 1, v0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-NEXT: v_lshrrev_b32_e32 v4, 2, v1
-; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_and_b32 v0, 1, v0
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX12-NEXT: v_dual_mov_b32 v3, v5 :: v_dual_and_b32 v4, 0xffff, v4
+; GFX12-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_and_b32 v2, 0xffff, v2
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-NEXT: v_and_b32_e32 v0, 0xffff, v0
+; GFX12-NEXT: v_and_b32_e32 v4, 0xffff, v4
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b64 v5, v[4:5], s[0:1] offset:16
; GFX12-NEXT: global_store_b128 v5, v[0:3], s[0:1]
@@ -5603,6 +5607,8 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v5, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, v5
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-FAKE16-NEXT: global_load_u8 v0, v5, s[2:3] nv
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -5610,7 +5616,7 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: v_bfe_u32 v2, v0, 1, 1
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX1250-FAKE16-NEXT: v_dual_lshrrev_b32 v4, 2, v1 :: v_dual_bitop2_b32 v0, 1, v0 bitop3:0x40
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v1, v5 :: v_dual_mov_b32 v3, v5
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, v5
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff, v0
@@ -5625,6 +5631,8 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-REAL16-NEXT: global_load_u8 v0, v1, s[2:3] nv
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
@@ -5638,7 +5646,7 @@ define amdgpu_kernel void @constant_zextload_v3i1_to_v3i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s3
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s4
; GFX1250-REAL16-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v2, s3
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
; GFX1250-REAL16-NEXT: global_store_b64 v1, v[0:1], s[0:1] offset:16
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s2
@@ -5865,6 +5873,8 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT: v_mov_b32_e32 v1, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_mov_b32_e32 v3, v1
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_d16_u8 v0, v1, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
@@ -5874,10 +5884,10 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX12-NEXT: v_lshrrev_b32_e32 v2, 3, v0
; GFX12-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX12-NEXT: v_dual_mov_b32 v0, s3 :: v_dual_mov_b32 v3, v1
+; GFX12-NEXT: v_mov_b32_e32 v0, s3
+; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10001
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-NEXT: v_and_b32_e32 v2, 0xffff, v2
-; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10001
; GFX12-NEXT: s_and_b32 s2, s2, 1
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_and_b32 s3, 0xffff, s3
@@ -5894,6 +5904,8 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v3, v1
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-FAKE16-NEXT: global_load_u8 v0, v1, s[2:3] nv
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
@@ -5903,7 +5915,7 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(SALU_CYCLE_1)
; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v2, 3, v0
; GFX1250-FAKE16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, s3
+; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v0, s3
; GFX1250-FAKE16-NEXT: s_bfe_u32 s3, s2, 0x10001
; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff, v2
@@ -5930,7 +5942,7 @@ define amdgpu_kernel void @constant_zextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-REAL16-NEXT: s_lshr_b32 s3, s3, 3
; GFX1250-REAL16-NEXT: s_and_b32 s4, 0xffff, s4
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s3
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v0, s4 :: v_dual_mov_b32 v3, v1
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v0, s4
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s3
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s2, 0x10001
; GFX1250-REAL16-NEXT: s_and_b32 s2, s2, 1
@@ -6073,17 +6085,17 @@ define amdgpu_kernel void @constant_sextload_v4i1_to_v4i64(ptr addrspace(1) %out
; GFX1250-NEXT: global_load_u8 v0, v9, s[2:3] nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_dual_lshrrev_b32 v2, 3, v0 :: v_dual_lshrrev_b32 v4, 2, v0
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX1250-NEXT: v_bfe_i32 v6, v2, 0, 1
; GFX1250-NEXT: v_lshrrev_b32_e32 v8, 1, v0
-; GFX1250-NEXT: v_bfe_i32 v4, v4, 0, 1
; GFX1250-NEXT: v_bfe_i32 v0, v0, 0, 1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX1250-NEXT: v_ashrrev_i32_e32 v7, 31, v6
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_4)
+; GFX1250-NEXT: v_bfe_i32 v6, v2, 0, 1
+; GFX1250-NEXT: v_bfe_i32 v4, v4, 0, 1
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX1250-NEXT: v_ashrrev_i32_e32 v1, 31, v0
; GFX1250-NEXT: v_bfe_i32 v2, v8, 0, 1
-; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX1250-NEXT: v_ashrrev_i32_e32 v5, 31, v4
-; GFX1250-NEXT: v_dual_ashrrev_i32 v1, 31, v0 :: v_dual_ashrrev_i32 v3, 31, v2
+; GFX1250-NEXT: v_dual_ashrrev_i32 v7, 31, v6 :: v_dual_ashrrev_i32 v5, 31, v4
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_2)
+; GFX1250-NEXT: v_ashrrev_i32_e32 v3, 31, v2
; GFX1250-NEXT: s_clause 0x1
; GFX1250-NEXT: global_store_b128 v9, v[4:7], s[0:1] offset:16
; GFX1250-NEXT: global_store_b128 v9, v[0:3], s[0:1]
@@ -6254,20 +6266,21 @@ define amdgpu_kernel void @constant_zextload_v8i1_to_v8i64(ptr addrspace(1) %out
; GFX1250-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v5, v1
+; GFX1250-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v9, v1
+; GFX1250-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v13, v1
; GFX1250-NEXT: s_wait_kmcnt 0x0
; GFX1250-NEXT: global_load_u8 v12, v1, s[2:3] nv
; GFX1250-NEXT: s_wait_loadcnt 0x0
; GFX1250-NEXT: v_and_b32_e32 v0, 0xffff, v12
+; GFX1250-NEXT: v_mov_b32_e32 v15, v1
; GFX1250-NEXT: v_bfe_u32 v6, v12, 5, 1
; GFX1250-NEXT: v_bfe_u32 v4, v12, 4, 1
; GFX1250-NEXT: v_bfe_u32 v10, v12, 3, 1
-; GFX1250-NEXT: v_bfe_u32 v8, v12, 2, 1
-; GFX1250-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_lshrrev_b32 v2, 7, v0
-; GFX1250-NEXT: v_mov_b32_e32 v5, v1
+; GFX1250-NEXT: v_lshrrev_b32_e32 v2, 7, v0
; GFX1250-NEXT: v_bfe_u32 v0, v0, 6, 1
-; GFX1250-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v9, v1
-; GFX1250-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v13, v1
-; GFX1250-NEXT: v_mov_b32_e32 v15, v1
+; GFX1250-NEXT: v_bfe_u32 v8, v12, 2, 1
; GFX1250-NEXT: v_bfe_u32 v14, v12, 1, 1
; GFX1250-NEXT: v_and_b32_e32 v12, 1, v12
; GFX1250-NEXT: s_clause 0x3
@@ -6458,21 +6471,22 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i64(ptr addrspace(1) %out
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
+; GFX12-NEXT: v_mov_b32_e32 v0, s2
; GFX12-NEXT: v_bfe_i32 v12, v9, 0, 1
; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX12-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
+; GFX12-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
+; GFX12-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_mov_b32 v4, s6
; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v5, s7
-; GFX12-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v7, s9
-; GFX12-NEXT: v_dual_mov_b32 v8, s10 :: v_dual_mov_b32 v9, s11
-; GFX12-NEXT: v_dual_mov_b32 v10, s12 :: v_dual_mov_b32 v11, s13
-; GFX12-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_mov_b32 v15, s15
+; GFX12-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
+; GFX12-NEXT: v_dual_mov_b32 v7, s9 :: v_dual_mov_b32 v8, s10
+; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s12
+; GFX12-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v14, s14
; GFX12-NEXT: v_ashrrev_i32_e32 v13, 31, v12
+; GFX12-NEXT: v_mov_b32_e32 v15, s15
; GFX12-NEXT: s_clause 0x3
; GFX12-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX12-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
@@ -6500,20 +6514,20 @@ define amdgpu_kernel void @constant_sextload_v8i1_to_v8i64(ptr addrspace(1) %out
; GFX1250-NEXT: s_lshr_b32 s14, s3, 1
; GFX1250-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
+; GFX1250-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-NEXT: v_bfe_i32 v12, v10, 0, 1
; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
-; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
+; GFX1250-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
+; GFX1250-NEXT: v_dual_mov_b32 v3, s5 :: v_dual_mov_b32 v4, s6
; GFX1250-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v5, s7
-; GFX1250-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v7, s9
-; GFX1250-NEXT: v_dual_mov_b32 v8, s10 :: v_dual_mov_b32 v9, s11
-; GFX1250-NEXT: v_dual_mov_b32 v10, s12 :: v_dual_mov_b32 v11, s13
-; GFX1250-NEXT: v_dual_mov_b32 v14, s14 :: v_dual_ashrrev_i32 v13, 31, v12
-; GFX1250-NEXT: v_mov_b32_e32 v15, s15
+; GFX1250-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
+; GFX1250-NEXT: v_dual_mov_b32 v7, s9 :: v_dual_mov_b32 v8, s10
+; GFX1250-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v10, s12
+; GFX1250-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v14, s14
+; GFX1250-NEXT: v_dual_ashrrev_i32 v13, 31, v12 :: v_dual_mov_b32 v15, s15
; GFX1250-NEXT: s_clause 0x3
; GFX1250-NEXT: global_store_b128 v16, v[0:3], s[0:1] offset:48
; GFX1250-NEXT: global_store_b128 v16, v[4:7], s[0:1] offset:32
@@ -6746,22 +6760,24 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12: ; %bb.0:
; GFX12-NEXT: s_load_b128 s[0:3], s[4:5], 0x24
; GFX12-NEXT: v_mov_b32_e32 v3, 0
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_mov_b32_e32 v5, v3
+; GFX12-NEXT: v_mov_b32_e32 v1, v3
+; GFX12-NEXT: v_mov_b32_e32 v7, v3
+; GFX12-NEXT: v_mov_b32_e32 v9, v3
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: global_load_d16_b16 v0, v3, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_readfirstlane_b32 s2, v0
+; GFX12-NEXT: v_and_b32_e32 v6, 0xffff, v0
; GFX12-NEXT: s_bfe_u32 s3, s2, 0x1000a
-; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000c
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_bfe_u32 v4, v6, 11, 1
; GFX12-NEXT: v_mov_b32_e32 v2, s3
-; GFX12-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_and_b32 v6, 0xffff, v0
; GFX12-NEXT: s_bfe_u32 s3, s2, 0x10009
-; GFX12-NEXT: v_mov_b32_e32 v1, v3
-; GFX12-NEXT: v_mov_b32_e32 v7, v3
-; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_3)
-; GFX12-NEXT: v_bfe_u32 v4, v6, 11, 1
; GFX12-NEXT: v_bfe_u32 v0, v6, 8, 1
+; GFX12-NEXT: s_bfe_u32 s4, s2, 0x1000c
; GFX12-NEXT: v_lshrrev_b32_e32 v8, 15, v6
-; GFX12-NEXT: v_mov_b32_e32 v9, v3
; GFX12-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: v_mov_b32_e32 v2, s3
@@ -6805,32 +6821,33 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-FAKE16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-FAKE16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v1, 0
+; GFX1250-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_mov_b32 v5, v1
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v7, v1 :: v_dual_mov_b32 v9, v1
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v13, v1
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v15, v1 :: v_dual_mov_b32 v17, v1
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v21, v1
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v23, v1 :: v_dual_mov_b32 v25, v1
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v27, v1 :: v_dual_mov_b32 v29, v1
; GFX1250-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX1250-FAKE16-NEXT: global_load_u16 v12, v1, s[2:3] nv
; GFX1250-FAKE16-NEXT: s_wait_loadcnt 0x0
+; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v31, v1 :: v_dual_bitop2_b32 v28, 1, v12 bitop3:0x40
; GFX1250-FAKE16-NEXT: v_and_b32_e32 v22, 0xffff, v12
; GFX1250-FAKE16-NEXT: v_bfe_u32 v0, v12, 10, 1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v6, v12, 9, 1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v14, v12, 13, 1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v18, v12, 7, 1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v2, v22, 11, 1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v3, v1 :: v_dual_lshrrev_b32 v10, 15, v22
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v5, v1 :: v_dual_mov_b32 v7, v1
-; GFX1250-FAKE16-NEXT: v_mov_b32_e32 v9, v1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v4, v22, 8, 1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v11, v1 :: v_dual_mov_b32 v13, v1
+; GFX1250-FAKE16-NEXT: v_lshrrev_b32_e32 v10, 15, v22
; GFX1250-FAKE16-NEXT: v_bfe_u32 v8, v22, 14, 1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v15, v1 :: v_dual_mov_b32 v17, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v31, v1 :: v_dual_bitop2_b32 v28, 1, v12 bitop3:0x40
; GFX1250-FAKE16-NEXT: v_bfe_u32 v26, v12, 3, 1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v30, v12, 1, 1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v24, v12, 2, 1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v20, v12, 4, 1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v16, v12, 6, 1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v12, v12, 12, 1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v19, v1 :: v_dual_mov_b32 v21, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v23, v1 :: v_dual_mov_b32 v25, v1
-; GFX1250-FAKE16-NEXT: v_dual_mov_b32 v27, v1 :: v_dual_mov_b32 v29, v1
; GFX1250-FAKE16-NEXT: v_bfe_u32 v22, v22, 5, 1
; GFX1250-FAKE16-NEXT: s_clause 0x7
; GFX1250-FAKE16-NEXT: global_store_b128 v1, v[0:3], s[0:1] offset:80
@@ -6848,6 +6865,8 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-REAL16-NEXT: s_setreg_imm32_b32 hwreg(HW_REG_WAVE_MODE, 25, 1), 1 ; msbs: dst=0 src0=0 src1=0 src2=0
; GFX1250-REAL16-NEXT: s_load_b128 s[0:3], s[4:5], 0x24 nv
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v3, 0
+; GFX1250-REAL16-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, v3
; GFX1250-REAL16-NEXT: s_wait_kmcnt 0x0
; GFX1250-REAL16-NEXT: global_load_u16 v0, v3, s[2:3] nv
; GFX1250-REAL16-NEXT: s_wait_loadcnt 0x0
@@ -6855,27 +6874,30 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-REAL16-NEXT: s_and_b32 s3, 0xffff, s2
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x1000a
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x1000b
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v5, v3
+; GFX1250-REAL16-NEXT: v_dual_mov_b32 v5, v3 :: v_dual_mov_b32 v2, s4
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10009
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x10008
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v1, v3
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:80
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-REAL16-NEXT: s_lshr_b32 s4, s3, 15
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s3, 0x1000e
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-REAL16-NEXT: s_lshr_b32 s4, s3, 15
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s3, 0x10005
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:64
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x1000d
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x1000c
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x1000d
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:112
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
-; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
-; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10007
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s5
; GFX1250-REAL16-NEXT: s_bfe_u32 s5, s2, 0x10006
+; GFX1250-REAL16-NEXT: v_mov_b32_e32 v4, s4
+; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10007
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:96
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s5 :: v_dual_mov_b32 v4, s4
@@ -6885,12 +6907,11 @@ define amdgpu_kernel void @constant_zextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
; GFX1250-REAL16-NEXT: s_bfe_u32 s3, s2, 0x10003
; GFX1250-REAL16-NEXT: s_bfe_u32 s4, s2, 0x10002
-; GFX1250-REAL16-NEXT: v_mov_b32_e32 v0, s2
; GFX1250-REAL16-NEXT: s_bfe_u32 s2, s2, 0x10001
+; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 1, v0
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:32
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v4, s3
-; GFX1250-REAL16-NEXT: v_and_b32_e32 v0, 1, v0
; GFX1250-REAL16-NEXT: global_store_b128 v3, v[2:5], s[0:1] offset:16
; GFX1250-REAL16-NEXT: s_wait_xcnt 0x0
; GFX1250-REAL16-NEXT: v_mov_b32_e32 v2, s2
@@ -7186,15 +7207,14 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12-NEXT: global_load_d16_b16 v0, v32, s[2:3]
; GFX12-NEXT: s_wait_loadcnt 0x0
; GFX12-NEXT: v_readfirstlane_b32 s3, v0
-; GFX12-NEXT: s_lshr_b32 s4, s3, 15
+; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX12-NEXT: v_mov_b32_e32 v28, s3
; GFX12-NEXT: s_lshr_b32 s2, s3, 14
-; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
-; GFX12-NEXT: s_delay_alu instid0(SALU_CYCLE_1)
-; GFX12-NEXT: v_dual_mov_b32 v28, s3 :: v_dual_mov_b32 v3, s5
-; GFX12-NEXT: s_lshr_b32 s6, s3, 12
+; GFX12-NEXT: s_lshr_b32 s4, s3, 15
; GFX12-NEXT: s_lshr_b32 s8, s3, 13
; GFX12-NEXT: s_lshr_b32 s10, s3, 10
; GFX12-NEXT: s_lshr_b32 s12, s3, 11
+; GFX12-NEXT: s_lshr_b32 s6, s3, 12
; GFX12-NEXT: s_lshr_b32 s14, s3, 8
; GFX12-NEXT: s_lshr_b32 s16, s3, 9
; GFX12-NEXT: s_lshr_b32 s18, s3, 6
@@ -7207,34 +7227,36 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX12-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
; GFX12-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
; GFX12-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
-; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
+; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
; GFX12-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
; GFX12-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
; GFX12-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
+; GFX12-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
; GFX12-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
; GFX12-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
; GFX12-NEXT: s_wait_alu depctr_sa_sdst(0)
-; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v5, s7
+; GFX12-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v3, s5
; GFX12-NEXT: v_dual_mov_b32 v1, s3 :: v_dual_mov_b32 v2, s4
-; GFX12-NEXT: v_dual_mov_b32 v7, s9 :: v_dual_mov_b32 v4, s6
-; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v6, s8
-; GFX12-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v8, s10
-; GFX12-NEXT: v_dual_mov_b32 v13, s15 :: v_dual_mov_b32 v10, s12
-; GFX12-NEXT: v_mov_b32_e32 v15, s17
+; GFX12-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
+; GFX12-NEXT: v_dual_mov_b32 v9, s11 :: v_dual_mov_b32 v8, s10
+; GFX12-NEXT: v_dual_mov_b32 v11, s13 :: v_dual_mov_b32 v10, s12
+; GFX12-NEXT: v_mov_b32_e32 v13, s15
; GFX12-NEXT: v_bfe_i32 v28, v28, 0, 1
; GFX12-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
; GFX12-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v17, s19
-; GFX12-NEXT: v_dual_mov_b32 v14, s16 :: v_dual_mov_b32 v19, s21
+; GFX12-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s9
+; GFX12-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v15, s17
+; GFX12-NEXT: v_dual_mov_b32 v14, s16 :: v_dual_mov_b32 v17, s19
; GFX12-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
; GFX12-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v21, s23
-; GFX12-NEXT: v_dual_mov_b32 v18, s20 :: v_dual_mov_b32 v23, s25
+; GFX12-NEXT: v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v19, s21
+; GFX12-NEXT: v_dual_mov_b32 v18, s20 :: v_dual_mov_b32 v21, s23
; GFX12-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX12-NEXT: v_dual_mov_b32 v20, s22 :: v_dual_mov_b32 v25, s27
-; GFX12-NEXT: v_dual_mov_b32 v22, s24 :: v_dual_mov_b32 v27, s29
-; GFX12-NEXT: v_dual_mov_b32 v24, s26 :: v_dual_mov_b32 v31, s31
-; GFX12-NEXT: v_mov_b32_e32 v26, s28
+; GFX12-NEXT: v_dual_mov_b32 v20, s22 :: v_dual_mov_b32 v23, s25
+; GFX12-NEXT: v_dual_mov_b32 v22, s24 :: v_dual_mov_b32 v25, s27
+; GFX12-NEXT: v_dual_mov_b32 v24, s26 :: v_dual_mov_b32 v27, s29
+; GFX12-NEXT: v_dual_mov_b32 v26, s28 :: v_dual_mov_b32 v31, s31
; GFX12-NEXT: v_mov_b32_e32 v30, s30
; GFX12-NEXT: s_clause 0x1
; GFX12-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
@@ -7262,10 +7284,10 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-NEXT: v_mov_b32_e32 v28, s3
; GFX1250-NEXT: s_lshr_b32 s2, s3, 14
; GFX1250-NEXT: s_lshr_b32 s4, s3, 15
+; GFX1250-NEXT: s_lshr_b32 s8, s3, 13
; GFX1250-NEXT: s_lshr_b32 s10, s3, 10
; GFX1250-NEXT: s_lshr_b32 s12, s3, 11
; GFX1250-NEXT: s_lshr_b32 s6, s3, 12
-; GFX1250-NEXT: s_lshr_b32 s8, s3, 13
; GFX1250-NEXT: s_lshr_b32 s14, s3, 8
; GFX1250-NEXT: s_lshr_b32 s16, s3, 9
; GFX1250-NEXT: s_lshr_b32 s18, s3, 6
@@ -7277,40 +7299,39 @@ define amdgpu_kernel void @constant_sextload_v16i1_to_v16i64(ptr addrspace(1) %o
; GFX1250-NEXT: s_lshr_b32 s30, s3, 1
; GFX1250-NEXT: s_bfe_i64 s[12:13], s[12:13], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[10:11], s[10:11], 0x10000
+; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[2:3], s[2:3], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[4:5], s[4:5], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[16:17], s[16:17], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[14:15], s[14:15], 0x10000
-; GFX1250-NEXT: s_bfe_i64 s[8:9], s[8:9], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[6:7], s[6:7], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[20:21], s[20:21], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[18:19], s[18:19], 0x10000
; GFX1250-NEXT: v_dual_mov_b32 v0, s2 :: v_dual_mov_b32 v1, s3
; GFX1250-NEXT: v_dual_mov_b32 v2, s4 :: v_dual_mov_b32 v3, s5
+; GFX1250-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v7, s9
; GFX1250-NEXT: v_dual_mov_b32 v8, s10 :: v_dual_mov_b32 v9, s11
; GFX1250-NEXT: v_dual_mov_b32 v10, s12 :: v_dual_mov_b32 v11, s13
+; GFX1250-NEXT: v_mov_b32_e32 v12, s14
; GFX1250-NEXT: v_bfe_i32 v28, v28, 0, 1
; GFX1250-NEXT: s_bfe_i64 s[24:25], s[24:25], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[22:23], s[22:23], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v4, s6 :: v_dual_mov_b32 v5, s7
-; GFX1250-NEXT: v_dual_mov_b32 v6, s8 :: v_dual_mov_b32 v7, s9
-; GFX1250-NEXT: v_dual_mov_b32 v12, s14 :: v_dual_mov_b32 v13, s15
-; GFX1250-NEXT: v_dual_mov_b32 v14, s16 :: v_dual_mov_b32 v15, s17
+; GFX1250-NEXT: v_dual_mov_b32 v5, s7 :: v_dual_mov_b32 v6, s8
+; GFX1250-NEXT: v_dual_mov_b32 v13, s15 :: v_dual_mov_b32 v14, s16
+; GFX1250-NEXT: v_dual_mov_b32 v15, s17 :: v_dual_mov_b32 v16, s18
; GFX1250-NEXT: s_bfe_i64 s[28:29], s[28:29], 0x10000
; GFX1250-NEXT: s_bfe_i64 s[26:27], s[26:27], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v16, s18 :: v_dual_mov_b32 v17, s19
-; GFX1250-NEXT: v_dual_mov_b32 v18, s20 :: v_dual_mov_b32 v19, s21
+; GFX1250-NEXT: v_dual_mov_b32 v17, s19 :: v_dual_mov_b32 v18, s20
+; GFX1250-NEXT: v_dual_mov_b32 v19, s21 :: v_dual_mov_b32 v20, s22
; GFX1250-NEXT: s_bfe_i64 s[30:31], s[30:31], 0x10000
-; GFX1250-NEXT: v_dual_mov_b32 v20, s22 :: v_dual_mov_b32 v21, s23
-; GFX1250-NEXT: v_dual_mov_b32 v22, s24 :: v_dual_mov_b32 v23, s25
-; GFX1250-NEXT: v_dual_mov_b32 v24, s26 :: v_dual_mov_b32 v25, s27
-; GFX1250-NEXT: v_dual_mov_b32 v26, s28 :: v_dual_mov_b32 v27, s29
-; GFX1250-NEXT: v_dual_mov_b32 v30, s30 :: v_dual_mov_b32 v31, s31
-; GFX1250-NEXT: s_clause 0x1
+; GFX1250-NEXT: v_dual_mov_b32 v21, s23 :: v_dual_mov_b32 v22, s24
+; GFX1250-NEXT: v_dual_mov_b32 v23, s25 :: v_dual_mov_b32 v24, s26
+; GFX1250-NEXT: v_dual_mov_b32 v25, s27 :: v_dual_mov_b32 v26, s28
+; GFX1250-NEXT: v_dual_mov_b32 v27, s29 :: v_dual_mov_b32 v30, s30
+; GFX1250-NEXT: v_dual_mov_b32 v31, s31 :: v_dual_ashrrev_i32 v29, 31, v28
+; GFX1250-NEXT: s_clause 0x7
; GFX1250-NEXT: global_store_b128 v32, v[0:3], s[0:1] offset:112
; GFX1250-NEXT: global_store_b128 v32, v[4:7], s[0:1] offset:96
-; GFX1250-NEXT: v_ashrrev_i32_e32 v29, 31, v28
-; GFX1250-NEXT: s_clause 0x5
; GFX1250-NEXT: global_store_b128 v32, v[8:11], s[0:1] offset:80
; GFX1250-NEXT: global_store_b128 v32, v[12:15], s[0:1] offset:64
; GFX1250-NEXT: global_store_b128 v32, v[16:19], s[0:1] offset:48
diff --git a/llvm/test/CodeGen/AMDGPU/madak.ll b/llvm/test/CodeGen/AMDGPU/madak.ll
index 0855f2c837339..8d40458a46087 100644
--- a/llvm/test/CodeGen/AMDGPU/madak.ll
+++ b/llvm/test/CodeGen/AMDGPU/madak.ll
@@ -272,9 +272,9 @@ define amdgpu_kernel void @madak_2_use_f32(ptr addrspace(1) noalias %out, ptr ad
; GFX11-MAD-NEXT: global_load_b32 v3, v0, s[2:3] offset:8 glc dlc
; GFX11-MAD-NEXT: s_waitcnt vmcnt(0)
; GFX11-MAD-NEXT: v_mul_f32_e32 v2, v1, v2
-; GFX11-MAD-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-MAD-NEXT: v_dual_mul_f32 v1, v1, v3 :: v_dual_add_f32 v2, 0x41200000, v2
-; GFX11-MAD-NEXT: v_add_f32_e32 v1, 0x41200000, v1
+; GFX11-MAD-NEXT: v_mul_f32_e32 v1, v1, v3
+; GFX11-MAD-NEXT: s_delay_alu instid0(VALU_DEP_1)
+; GFX11-MAD-NEXT: v_dual_add_f32 v1, 0x41200000, v1 :: v_dual_add_f32 v2, 0x41200000, v2
; GFX11-MAD-NEXT: global_store_b32 v0, v2, s[0:1] dlc
; GFX11-MAD-NEXT: s_waitcnt_vscnt null, 0x0
; GFX11-MAD-NEXT: global_store_b32 v0, v1, s[2:3] offset:4 dlc
diff --git a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
index 5add8082bd81a..c253f42e0d3c8 100644
--- a/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
+++ b/llvm/test/CodeGen/AMDGPU/scratch-simple.ll
@@ -977,10 +977,7 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -992,14 +989,16 @@ define amdgpu_ps float @ps_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -1982,10 +1981,7 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -1997,14 +1993,16 @@ define amdgpu_vs float @vs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -2987,10 +2985,7 @@ define amdgpu_cs float @cs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -3002,14 +2997,16 @@ define amdgpu_cs float @cs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -3989,10 +3986,7 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -4004,14 +3998,16 @@ define amdgpu_hs float @hs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -4991,10 +4987,7 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -5006,14 +4999,16 @@ define amdgpu_gs float @gs_main(i32 %idx) {
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -6004,10 +5999,7 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -6019,14 +6011,16 @@ define amdgpu_hs <{i32, i32, i32, float}> @hs_ir_uses_scratch_offset(i32 inreg,
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -7016,10 +7010,7 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v18, v17 :: v_dual_and_b32 v33, 0x1fc, v0
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, 0xbefcd8a3 :: v_dual_mov_b32 v15, v11
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v1, 0xb7043519 :: v_dual_mov_b32 v10, v13
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v2, 0xbe31934f
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v32, 0x3f3d349c :: v_dual_mov_b32 v31, v11
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v2, 0xbe31934f :: v_dual_mov_b32 v31, v11
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v0, 0xb702e758
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v22, 0xbf638e39
; GFX11-FLATSCR-NEXT: s_clause 0x1
@@ -7031,14 +7022,16 @@ define amdgpu_gs <{i32, i32, i32, float}> @gs_ir_uses_scratch_offset(i32 inreg,
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[0:3], off offset:256
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[21:24], off offset:208
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[13:16], off offset:192
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v4, 0x3f20e7f4
; GFX11-FLATSCR-NEXT: scratch_load_b32 v0, v33, off
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v29, 0xbf523be1
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v32, 0x3f3d349c
; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v24, v21
-; GFX11-FLATSCR-NEXT: s_clause 0x1
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[4:7], off offset:832
+; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v7, 0x3703c499 :: v_dual_mov_b32 v16, v28
; GFX11-FLATSCR-NEXT: scratch_store_b128 off, v[29:32], off offset:784
-; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v7, 0x3703c499
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v31, 0xbf5f2ee2 :: v_dual_mov_b32 v32, v26
-; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v16, v28 :: v_dual_mov_b32 v23, v25
+; GFX11-FLATSCR-NEXT: v_mov_b32_e32 v23, v25
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v25, v26 :: v_dual_mov_b32 v18, v12
; GFX11-FLATSCR-NEXT: v_dual_mov_b32 v20, v17 :: v_dual_mov_b32 v21, v3
; GFX11-FLATSCR-NEXT: s_clause 0x1
diff --git a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
index 6bf6d540299f1..10c2162f5eadb 100644
--- a/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
+++ b/llvm/test/CodeGen/AMDGPU/vector_shuffle.packed.ll
@@ -5457,77 +5457,77 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
; GFX11-TRUE16-NEXT: global_load_b64 v[0:1], v6, s[0:1]
; GFX11-TRUE16-NEXT: global_load_b64 v[2:3], v6, s[4:5]
; GFX11-TRUE16-NEXT: global_load_b64 v[4:5], v6, s[6:7]
+; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v10, 0xffff0000, v5
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v5, 16, v5
; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v9, 16, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v11, 16, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-TRUE16-NEXT: v_dual_fmac_f32 v1, v12, v4 :: v_dual_lshlrev_b32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_dual_fmac_f32 v11, v12, v9 :: v_dual_lshlrev_b32 v8, 16, v2
+; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-TRUE16-NEXT: v_dual_fmac_f32 v1, v12, v4 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_bfe_u32 v15, v1, 16, 1
+; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v7, v8, v9
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v0, v8, v4
; GFX11-TRUE16-NEXT: v_or_b32_e32 v16, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_add3_u32 v15, v15, v1, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-TRUE16-NEXT: v_lshlrev_b32_e32 v7, 16, v0
-; GFX11-TRUE16-NEXT: v_dual_fmac_f32 v7, v8, v9 :: v_dual_and_b32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v0, v8, v4
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v4, v7, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
; GFX11-TRUE16-NEXT: v_add3_u32 v4, v4, v7, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v4, v2, v5
-; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v11, v12, v9
-; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v13, v11, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
-; GFX11-TRUE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
-; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc_lo
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_dual_fmac_f32 v4, v2, v5 :: v_dual_cndmask_b32 v1, v15, v16
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v0, v2, v10
+; GFX11-TRUE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v4, v4
-; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-TRUE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
-; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
-; GFX11-TRUE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
-; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v7, v3, v5
+; GFX11-TRUE16-NEXT: v_dual_fmac_f32 v7, v3, v5 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v0, v2, v10
; GFX11-TRUE16-NEXT: v_bfe_u32 v5, v4, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_bfe_u32 v2, v0, 16, 1
; GFX11-TRUE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v5, v8 :: v_dual_fmac_f32 v1, v3, v10
-; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-TRUE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-TRUE16-NEXT: v_dual_cndmask_b32 v4, v5, v8 :: v_dual_and_b32 v1, 0xffff0000, v1
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_3)
+; GFX11-TRUE16-NEXT: v_fmac_f32_e32 v1, v3, v10
+; GFX11-TRUE16-NEXT: v_bfe_u32 v10, v7, 16, 1
; GFX11-TRUE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-TRUE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v10, v10, v7, 0x7fff
; GFX11-TRUE16-NEXT: v_or_b32_e32 v11, 0x400000, v1
-; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX11-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v5, v10, v12, vcc_lo
; GFX11-TRUE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
; GFX11-TRUE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
@@ -5551,75 +5551,77 @@ define amdgpu_kernel void @fma_shuffle_v2bf16(ptr addrspace(1) nocapture readonl
; GFX11-FAKE16-NEXT: global_load_b64 v[0:1], v6, s[0:1]
; GFX11-FAKE16-NEXT: global_load_b64 v[2:3], v6, s[4:5]
; GFX11-FAKE16-NEXT: global_load_b64 v[4:5], v6, s[6:7]
+; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(2)
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX11-FAKE16-NEXT: s_waitcnt vmcnt(0)
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v10, 16, v5
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
; GFX11-FAKE16-NEXT: v_and_b32_e32 v5, 0xffff0000, v5
; GFX11-FAKE16-NEXT: v_and_b32_e32 v9, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v12, 16, v3
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v8, 16, v2
; GFX11-FAKE16-NEXT: v_and_b32_e32 v3, 0xffff0000, v3
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v11, v12, v9
+; GFX11-FAKE16-NEXT: v_bfe_u32 v13, v11, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v4, 16, v4
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v11, 0xffff0000, v1
-; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v1, 16, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_dual_fmac_f32 v1, v12, v4 :: v_dual_lshlrev_b32 v8, 16, v2
+; GFX11-FAKE16-NEXT: v_dual_fmac_f32 v1, v12, v4 :: v_dual_and_b32 v2, 0xffff0000, v2
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
+; GFX11-FAKE16-NEXT: v_lshlrev_b32_e32 v0, 16, v0
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_bfe_u32 v15, v1, 16, 1
+; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v7, v8, v9
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
+; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v0, v8, v4
; GFX11-FAKE16-NEXT: v_or_b32_e32 v16, 0x400000, v1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_2) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_add3_u32 v15, v15, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v2, 0xffff0000, v2
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v0
-; GFX11-FAKE16-NEXT: v_dual_fmac_f32 v7, v8, v9 :: v_dual_lshlrev_b32 v0, 16, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v0, v8, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-FAKE16-NEXT: v_bfe_u32 v4, v7, 16, 1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v7
+; GFX11-FAKE16-NEXT: v_bfe_u32 v9, v0, 16, 1
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v7, v7
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
; GFX11-FAKE16-NEXT: v_add3_u32 v4, v4, v7, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v4, v4, v8, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
-; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v4, v2, v5
-; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v11, v12, v9
-; GFX11-FAKE16-NEXT: v_bfe_u32 v9, v0, 16, 1
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
-; GFX11-FAKE16-NEXT: v_bfe_u32 v13, v11, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_add3_u32 v9, v9, v0, 0x7fff
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v14, 0x400000, v11
-; GFX11-FAKE16-NEXT: v_add3_u32 v13, v13, v11, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_4) | instid1(VALU_DEP_1)
; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v9, v12, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v15, v16, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v11, v11
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v7, v13, v14, vcc_lo
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v1, v3, v10
; GFX11-FAKE16-NEXT: v_and_b32_e32 v7, 0xffff0000, v7
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: v_dual_fmac_f32 v7, v3, v5 :: v_dual_and_b32 v0, 0xffff0000, v0
+; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v0, v2, v10
; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
-; GFX11-FAKE16-NEXT: v_bfe_u32 v9, v1, 16, 1
-; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v7, v3, v5
-; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_4)
-; GFX11-FAKE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
-; GFX11-FAKE16-NEXT: v_and_b32_e32 v0, 0xffff0000, v0
; GFX11-FAKE16-NEXT: v_bfe_u32 v11, v7, 16, 1
+; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v1, 0xffff0000, v1
; GFX11-FAKE16-NEXT: v_or_b32_e32 v12, 0x400000, v7
-; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
-; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v0, v2, v10
-; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add3_u32 v11, v11, v7, 0x7fff
-; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v1, v3, v10
; GFX11-FAKE16-NEXT: v_or_b32_e32 v3, 0x400000, v0
-; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v0, v0
-; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_bfe_u32 v9, v1, 16, 1
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v10, 0x400000, v1
+; GFX11-FAKE16-NEXT: v_add3_u32 v9, v9, v1, 0x7fff
+; GFX11-FAKE16-NEXT: v_and_b32_e32 v4, 0xffff0000, v4
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
+; GFX11-FAKE16-NEXT: v_fmac_f32_e32 v4, v2, v5
+; GFX11-FAKE16-NEXT: v_bfe_u32 v2, v0, 16, 1
+; GFX11-FAKE16-NEXT: v_bfe_u32 v5, v4, 16, 1
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_add3_u32 v2, v2, v0, 0x7fff
+; GFX11-FAKE16-NEXT: v_or_b32_e32 v8, 0x400000, v4
+; GFX11-FAKE16-NEXT: v_add3_u32 v5, v5, v4, 0x7fff
+; GFX11-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_3)
; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v0, v2, v3, vcc_lo
; GFX11-FAKE16-NEXT: v_cmp_u_f32_e32 vcc_lo, v1, v1
; GFX11-FAKE16-NEXT: v_cndmask_b32_e32 v1, v9, v10, vcc_lo
More information about the llvm-commits
mailing list